From 9bbe18834770125800641c03185504785cf4f3a9 Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Thu, 16 Jan 2025 16:34:36 +1100 Subject: [PATCH] pushlog: add basic models (bug 1940610) --- pyproject.toml | 1 + src/lando/main/admin.py | 11 ++ .../migrations/0001_initial_updated.py | 140 ++++++++++++++ src/lando/pushlog/migrations/__init__.py | 0 src/lando/pushlog/models/__init__.py | 9 + src/lando/pushlog/models/commit.py | 75 +++++++ src/lando/pushlog/models/consts.py | 5 + src/lando/pushlog/models/push.py | 63 ++++++ src/lando/pushlog/tests/test_models.py | 183 ++++++++++++++++++ src/lando/settings.py | 1 + 10 files changed, 488 insertions(+) create mode 100644 src/lando/pushlog/migrations/0001_initial_updated.py create mode 100644 src/lando/pushlog/migrations/__init__.py create mode 100644 src/lando/pushlog/models/__init__.py create mode 100644 src/lando/pushlog/models/commit.py create mode 100644 src/lando/pushlog/models/consts.py create mode 100644 src/lando/pushlog/models/push.py create mode 100644 src/lando/pushlog/tests/test_models.py diff --git a/pyproject.toml b/pyproject.toml index 45ea86bb..520cea15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ testpaths = [ "src/lando/api", "src/lando/dockerflow", "src/lando/main/tests", + "src/lando/pushlog/tests", "src/lando/tests", "src/lando/ui/tests", ] diff --git a/src/lando/main/admin.py b/src/lando/main/admin.py index d3ae3b0b..4db9a915 100644 --- a/src/lando/main/admin.py +++ b/src/lando/main/admin.py @@ -9,6 +9,12 @@ RevisionLandingJob, Worker, ) +from lando.pushlog.models import ( + Commit, + File, + Push, + Tag, +) admin.site.site_title = gettext_lazy("Lando Admin") admin.site.site_header = gettext_lazy("Lando Administration") @@ -43,3 +49,8 @@ class LandingJobAdmin(admin.ModelAdmin): admin.site.register(Repo, admin.ModelAdmin) admin.site.register(Worker, admin.ModelAdmin) admin.site.register(ConfigurationVariable, admin.ModelAdmin) + +admin.site.register(Push, admin.ModelAdmin) +admin.site.register(Commit, admin.ModelAdmin) +admin.site.register(File, admin.ModelAdmin) +admin.site.register(Tag, admin.ModelAdmin) diff --git a/src/lando/pushlog/migrations/0001_initial_updated.py b/src/lando/pushlog/migrations/0001_initial_updated.py new file mode 100644 index 00000000..67cb77f2 --- /dev/null +++ b/src/lando/pushlog/migrations/0001_initial_updated.py @@ -0,0 +1,140 @@ +# Generated by Django 5.1.4 on 2025-01-17 05:32 + +import django.core.validators +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("main", "0013_alter_repo_scm_type_alter_worker_scm"), + ("main", "0013_alter_repo_scm_type_alter_worker_scm"), + ("main", "0013_alter_repo_scm_type_alter_worker_scm"), + ] + + operations = [ + migrations.CreateModel( + name="File", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=4096)), + ( + "repo", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, to="main.repo" + ), + ), + ], + options={ + "unique_together": {("repo", "name")}, + }, + ), + migrations.CreateModel( + name="Commit", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "hash", + models.CharField( + db_index=True, + max_length=160, + validators=[ + django.core.validators.MaxLengthValidator(160), + django.core.validators.MinLengthValidator(160), + django.core.validators.RegexValidator("^([a-fA-F0-9])+"), + ], + ), + ), + ( + "repo", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, to="main.repo" + ), + ), + ("author", models.CharField(max_length=512)), + ("desc", models.TextField()), + ("parents", models.ManyToManyField(blank=True, to="pushlog.commit")), + ("files", models.ManyToManyField(to="pushlog.file")), + ], + options={ + "unique_together": {("repo", "hash")}, + }, + ), + migrations.CreateModel( + name="Push", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("push_id", models.PositiveIntegerField()), + ( + "repo", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, to="main.repo" + ), + ), + ("date", models.DateField(auto_now_add=True)), + ("user", models.EmailField(max_length=320)), + ("commits", models.ManyToManyField(to="pushlog.commit")), + ("branch", models.CharField(max_length=255)), + ], + options={ + "unique_together": {("push_id", "repo")}, + }, + ), + migrations.CreateModel( + name="Tag", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "repo", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, to="main.repo" + ), + ), + ( + "commit", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="pushlog.commit" + ), + ), + ], + options={ + "unique_together": {("repo", "name")}, + }, + ), + ] diff --git a/src/lando/pushlog/migrations/__init__.py b/src/lando/pushlog/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/lando/pushlog/models/__init__.py b/src/lando/pushlog/models/__init__.py new file mode 100644 index 00000000..a6418a53 --- /dev/null +++ b/src/lando/pushlog/models/__init__.py @@ -0,0 +1,9 @@ +from .commit import Commit, File, Tag +from .push import Push + +__all__ = [ + "Commit", + "File", + "Push", + "Tag", +] diff --git a/src/lando/pushlog/models/commit.py b/src/lando/pushlog/models/commit.py new file mode 100644 index 00000000..f653fc89 --- /dev/null +++ b/src/lando/pushlog/models/commit.py @@ -0,0 +1,75 @@ +from django.core.validators import ( + MaxLengthValidator, + MinLengthValidator, + RegexValidator, +) +from django.db import models + +from lando.main.models import Repo + +from .consts import COMMIT_ID_HEX_LENGTH, MAX_FILENAME_LENGTH, MAX_PATH_LENGTH + + +class File(models.Model): + name = models.CharField(max_length=MAX_PATH_LENGTH) + + repo = models.ForeignKey( + Repo, + # We don't want to delete the PushLog, even if we were to delete the repo + # object. + on_delete=models.DO_NOTHING, + ) + + class Meta: + unique_together = ("repo", "name") + + +class Commit(models.Model): + hash = models.CharField( + max_length=COMMIT_ID_HEX_LENGTH, + db_index=True, + blank=False, + validators=[ + MaxLengthValidator(COMMIT_ID_HEX_LENGTH), + MinLengthValidator(COMMIT_ID_HEX_LENGTH), + RegexValidator(r"^([a-fA-F0-9])+"), + ], + ) + + repo = models.ForeignKey( + Repo, + # We don't want to delete the PushLog, even if we were to delete the repo + # object. + on_delete=models.DO_NOTHING, + ) + + # Assuming a max email address length (see Push model), and then some space for a long name. + author = models.CharField(max_length=512) + + desc = models.TextField() + + files = models.ManyToManyField(File) + + parents = models.ManyToManyField("self", blank=True) + + class Meta: + unique_together = ("repo", "hash") + + def __repr__(self): + return f"<{self.__class__.__name__}({self.hash})>" + + +class Tag(models.Model): + # Tag names are limited by how long a filename the filesystem support. + name = models.CharField(max_length=MAX_FILENAME_LENGTH) + commit = models.ForeignKey(Commit, on_delete=models.CASCADE) + + repo = models.ForeignKey( + Repo, + # We don't want to delete the PushLog, even if we were to delete the repo + # object. + on_delete=models.DO_NOTHING, + ) + + class Meta: + unique_together = ("repo", "name") diff --git a/src/lando/pushlog/models/consts.py b/src/lando/pushlog/models/consts.py new file mode 100644 index 00000000..ad81433e --- /dev/null +++ b/src/lando/pushlog/models/consts.py @@ -0,0 +1,5 @@ +COMMIT_ID_HEX_LENGTH = 160 + +# Those are fairly common values under Unix. +MAX_FILENAME_LENGTH = 255 +MAX_PATH_LENGTH = 4096 diff --git a/src/lando/pushlog/models/push.py b/src/lando/pushlog/models/push.py new file mode 100644 index 00000000..e3183513 --- /dev/null +++ b/src/lando/pushlog/models/push.py @@ -0,0 +1,63 @@ +from django.db import models + +from lando.main.models import Repo + +from .commit import Commit +from .consts import MAX_FILENAME_LENGTH + +PUSH_SCM_TYPE_GIT = "git" +PUSH_SCM_TYPES = [PUSH_SCM_TYPE_GIT] + + +class Push(models.Model): + push_id = models.PositiveIntegerField() + + repo = models.ForeignKey( + Repo, + # We don't want to delete the PushLog, even if we were to delete the repo + # object. + on_delete=models.DO_NOTHING, + ) + + date = models.DateField( + auto_now=False, + auto_now_add=True, + ) + + # Maximum total lengths are defined in RFC-5321 [0]: 64 for the local-part, and 255 + # for the domain. + # [0] https://datatracker.ietf.org/doc/html/rfc5321#section-4.5.3.1.1 + user = models.EmailField(max_length=64 + 1 + 255) + + commits = models.ManyToManyField(Commit) + + # Branch names are limited by how long a filename the filesystem support. This is + # generally 255 bytes. + branch = models.CharField(max_length=MAX_FILENAME_LENGTH) + + class Meta: + unique_together = ("push_id", "repo") + + def __repr__(self): + return f"<{self.__class__.__name__}({self.push_id } on {self.repo})>" + + def save(self, *args, **kwargs): + if not self.id: + # Determine the next push_id on first save + next_push_id = self._next_push_id(self.repo) + self.push_id = next_push_id + super(Push, self).save(*args, **kwargs) + + @classmethod + def _next_push_id(cls, repo: Repo): + """Generate a monotonically increasing sequence of push_id, scoped by Repo.""" + max_push_id = ( + cls.objects.filter(repo=repo) + .order_by("-push_id") + .values_list("push_id", flat=True) + ) + + if max_push_id: + return max_push_id[0] + 1 + + return 1 diff --git a/src/lando/pushlog/tests/test_models.py b/src/lando/pushlog/tests/test_models.py new file mode 100644 index 00000000..d15abfcb --- /dev/null +++ b/src/lando/pushlog/tests/test_models.py @@ -0,0 +1,183 @@ +import pytest +from django.db.utils import IntegrityError + +from lando.main.models import Repo +from lando.pushlog.models import Commit, File, Push, Tag + + +@pytest.fixture +def make_repo(): + def repo_factory(seqno: int): + "Create a non-descript repository with a sequence number in the test DB." + return Repo.objects.create(name=f"repo-{seqno}", scm_type="git") + + return repo_factory + + +@pytest.fixture +def make_commit(): + def commit_factory(repo: Repo, seqno: int, message=None): + "Create a non-descript commit with a sequence number in the test DB." + if not message: + message = f"Commit {seqno}" + + return Commit.objects.create( + # Create a 160-character string, of which the first 8 bytes represent the seqno + # in decimal representation. + hash=(str(seqno).zfill(8) + "f" + 151 * "0"), + repo=repo, + author=f"author-{seqno}", + desc=message, + ) + + return commit_factory + + +@pytest.fixture +def make_file(): + def file_factory(repo: Repo, seqno: int): + "Create a non-descript file with a sequence number in the test DB." + return File.objects.create( + repo=repo, + name=f"file-{seqno}", + ) + + return file_factory + + +@pytest.fixture +def make_tag(): + def tag_factory(repo: Repo, seqno: int, commit: Commit): + "Create a non-descript tag with a sequence number in the test DB." + return Tag.objects.create( + repo=repo, + name=f"tag-{seqno}", + commit=commit, + ) + + return tag_factory + + +@pytest.fixture +def make_push(): + def push_factory(repo: Repo, commits: list[Commit]): + "Create a non-descript push containing the associated commits in the test DB." + push = Push.objects.create(repo=repo) + for c in commits: + push.commits.add(c) + push.save() + + return push + + return push_factory + + +@pytest.mark.django_db() +def test__pushlog__models__Commit(make_repo, make_commit, make_file): + # Model.objects.create() creates _and saves_ the object + repo = make_repo(1) + commit = make_commit(repo, 1) + + file1 = make_file(repo, 1) + file2 = make_file(repo, 2) + + commit.files.add(file1) + commit.files.add(file2) + commit.save() + + retrieved_commit = Commit.objects.get(hash=commit.hash) + + assert commit.hash in repr(commit) + assert retrieved_commit.id == commit.id + assert retrieved_commit.files.count() == 2 + + +@pytest.mark.django_db() +def test__pushlog__models__Commit_unique(make_repo, make_commit): + # Model.objects.create() creates _and saves_ the object + repo = make_repo(1) + make_commit(repo, 1) + + with pytest.raises( + IntegrityError, + match=r"duplicate key.*pushlog_commit_repo_id", + ): + make_commit(repo, 1) + + +@pytest.mark.django_db() +def test__pushlog__models__File_unique(make_repo, make_file): + # Model.objects.create() creates _and saves_ the object + repo = make_repo(1) + make_file(repo, 1) + + with pytest.raises( + IntegrityError, + match=r"duplicate key.*pushlog_file_repo_id", + ): + make_file(repo, 1) + + +@pytest.mark.django_db() +def test__pushlog__models__Tag(make_repo, make_commit, make_tag): + repo = make_repo(1) + commit = make_commit(repo, 1) + + tag1 = make_tag(repo, 1, commit) + make_tag(repo, 2, commit) + + retrieved_tags = Tag.objects.filter(commit=commit) + rtag1 = retrieved_tags.get(name="tag-1") + + assert retrieved_tags.count() == 2 + assert rtag1.id == tag1.id + + +@pytest.mark.django_db() +def test__pushlog__models__Tag_unique(make_repo, make_commit, make_tag): + # Model.objects.create() creates _and saves_ the object + repo = make_repo(1) + commit = make_commit(repo, 1) + make_tag(repo, 1, commit) + + with pytest.raises( + IntegrityError, + match=r"duplicate key.*pushlog_tag_repo_id", + ): + make_tag(repo, 1, commit) + + +@pytest.mark.django_db() +def test__pushlog__models__Pushlog(make_repo, make_commit, make_push): + repo1 = make_repo(1) + repo2 = make_repo(2) + + commit11 = make_commit(repo1, 1) + commit12 = make_commit(repo1, 2) + push11 = make_push(repo1, [commit11, commit12]) + + commit21 = make_commit(repo2, 1) + commit22 = make_commit(repo2, 2) + push21 = make_push(repo2, [commit21, commit22]) + + commit13 = make_commit(repo1, 3) + commit14 = make_commit(repo1, 4) + push12 = make_push(repo1, [commit13, commit14]) + + push11_repr = repr(push11) + assert f"({push11.push_id}" in push11_repr + assert str(repo1) in push11_repr + + # Ensure that the push_id are scoped by repo. + assert push11.push_id == 1 + assert ( + push21.push_id == 1 + ), "first push_id on second repository is not a strict incrementation" + assert ( + push12.push_id == 2 + ), "second push_id on first repository is not a strict incrementation" + + push12.save() + assert ( + push12.push_id == 2 + ), "second push_id on first repository has changed on re-save" diff --git a/src/lando/settings.py b/src/lando/settings.py index 62186812..d98a7c8a 100644 --- a/src/lando/settings.py +++ b/src/lando/settings.py @@ -47,6 +47,7 @@ "compressor", "mozilla_django_oidc", "lando.main", + "lando.pushlog", "lando.utils", "lando.api", "lando.dockerflow",