landing_worker: Port Hg Landing Worker (Bug 1869035)

.gitignore

@@ -2,6 +2,8 @@
 *.pyc
 .env
 .pytest_cache
+.idea
+*.bak
 /*.egg-info/
 __pycache__
 env

Dockerfile

@@ -3,14 +3,25 @@ FROM python:3.12
 EXPOSE 80
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE 1
-RUN adduser --system --no-create-home app
+
+RUN addgroup --gid 10001 app \
+    && adduser \
+        --disabled-password \
+        --uid 10001 \
+        --gid 10001 \
+        --home /app \
+        --gecos "app,,," \
+        app
 
 RUN mkdir /code
 COPY ./ /code
 
 RUN mkdir -p /code/.ruff_cache
 RUN chown -R app /code/.ruff_cache
 
+RUN mkdir -p /files/repos
+RUN chown -R app /files/repos
+
 RUN pip install --upgrade pip
 
 

compose.yaml

@@ -27,7 +27,32 @@ services:
     volumes:
       - ./:/code
       - ./staticfiles:/staticfiles
-      - media:/mediafiles
+      - media:/files/
+    env_file:
+      - .env
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+
+  hg-landing-worker:
+    build:
+      context: .
+    stdin_open: true
+    tty: true
+    image: lando 
+    command: bash -c "
+      lando generate_version_file &&
+      lando migrate &&
+      lando collectstatic --clear --no-input &&
+      lando ensure_access_groups &&
+      lando ensure_workers &&
+      lando hg_landing_worker"
+    volumes:
+      - ./:/code
+      - ./staticfiles:/staticfiles
+      - media:/files/
     env_file:
       - .env
     depends_on:
@@ -44,7 +69,6 @@ services:
       - lando
     volumes:
       - ./staticfiles:/staticfiles
-      - media:/mediafiles
 
   redis:
     image: redis:7.2

nginx/nginx.conf

@@ -32,8 +32,4 @@ server {
     location /static {
         alias /staticfiles/;
     }
-
-    location /media {
-        alias /mediafiles/;
-    }
 }

requirements.txt

@@ -16,3 +16,4 @@ rs-parsepatch
 ruff
 setuptools-scm
 uwsgi
+django-storages[google]

src/lando/api/legacy/api/__init__.py

@@ -1,8 +1,6 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from lando.api.legacy.api import stacks, transplants
 
-
-def get():
-    """Return a redirect repsonse to the swagger specification."""
-    return None, 302, {"Location": "/swagger.json"}
+__all__ = ["stacks", "transplants"]

src/lando/api/legacy/api/landing_jobs.py

@@ -3,15 +3,13 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import logging
 
-from lando.api.legacy import auth
 from lando.main.models.landing_job import LandingJob, LandingJobAction, LandingJobStatus
-from lando.main.support import ProblemException, g
+from lando.main.support import ProblemException
 
 logger = logging.getLogger(__name__)
 
 
-@auth.require_auth0(scopes=("lando", "profile", "email"), userinfo=True)
-def put(landing_job_id: str, data: dict):
+def put(request, landing_job_id: str, data: dict):
     """Update a landing job.
 
     Checks whether the logged in user is allowed to modify the landing job that is
@@ -29,6 +27,8 @@ def put(landing_job_id: str, data: dict):
             updated (for example, when trying to cancel a job that is already in
             progress).
     """
+    if not request.user.is_authenticated:
+        raise PermissionError
     with LandingJob.lock_table:
         landing_job = LandingJob.objects.get(pk=landing_job_id)
 
@@ -40,7 +40,7 @@ def put(landing_job_id: str, data: dict):
             type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404",
         )
 
-    ldap_username = g.auth0_user.email
+    ldap_username = request.user.email
     if landing_job.requester_email != ldap_username:
         raise ProblemException(
             403,

src/lando/api/legacy/api/stacks.py

@@ -5,6 +5,7 @@
 import urllib.parse
 
 from django.conf import settings
+from django.http import Http404
 
 from lando.api.legacy.commit_message import format_commit_message
 from lando.api.legacy.decorators import require_phabricator_api_key
@@ -15,7 +16,6 @@
     get_secure_project_phid,
     project_search,
 )
-from lando.api.legacy.repos import get_repos_for_env
 from lando.api.legacy.reviews import (
     approvals_for_commit_message,
     get_collated_reviewers,
@@ -41,39 +41,33 @@
 from lando.api.legacy.users import user_search
 from lando.api.legacy.validation import revision_id_to_int
 from lando.main.models.revision import Revision
-from lando.main.support import problem
+from lando.main.util import get_repos_for_env
 
 logger = logging.getLogger(__name__)
 
-not_found_problem = problem(
-    404,
-    "Revision not found",
-    "The requested revision does not exist",
-    type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404",
-)
-
 
 @require_phabricator_api_key(optional=True)
-def get(phab: PhabricatorClient, revision_id: str):
+def get(phab: PhabricatorClient, request, revision_id: str):
     """Get the stack a revision is part of.
 
     Args:
         revision_id: (string) ID of the revision in 'D{number}' format
     """
+    revision_id = f"D{revision_id}"
     revision_id_int = revision_id_to_int(revision_id)
 
     revision = phab.call_conduit(
         "differential.revision.search", constraints={"ids": [revision_id_int]}
     )
     revision = phab.single(revision, "data", none_when_empty=True)
     if revision is None:
-        return not_found_problem
+        raise Http404
 
     nodes, edges = build_stack_graph(revision)
     try:
         stack_data = request_extended_revision_data(phab, list(nodes))
     except ValueError:
-        return not_found_problem
+        raise Http404
 
     supported_repos = get_repos_for_env(settings.ENVIRONMENT)
     landable_repos = get_landable_repos_for_revision_data(stack_data, supported_repos)

src/lando/api/legacy/api/transplants.py

@@ -8,8 +8,9 @@
 
 import kombu
 from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.exceptions import PermissionDenied
 
-from lando.api.legacy import auth
 from lando.api.legacy.commit_message import format_commit_message
 from lando.api.legacy.decorators import require_phabricator_api_key
 from lando.api.legacy.phabricator import PhabricatorClient
@@ -23,10 +24,6 @@
     get_testing_tag_project_phids,
     project_search,
 )
-from lando.api.legacy.repos import (
-    Repo,
-    get_repos_for_env,
-)
 from lando.api.legacy.reviews import (
     approvals_for_commit_message,
     get_approved_by_ids,
@@ -59,13 +56,16 @@
     parse_landing_path,
     revision_id_to_int,
 )
+from lando.main.config.repos import RepoTypeEnum
 from lando.main.models.landing_job import (
     LandingJob,
     LandingJobStatus,
     add_revisions_to_job,
 )
+from lando.main.models.repo import Repo
 from lando.main.models.revision import Revision
-from lando.main.support import ProblemException, g, problem
+from lando.main.support import ProblemException, problem
+from lando.main.util import get_repos_for_env
 from lando.utils.tasks import admin_remove_phab_project
 
 logger = logging.getLogger(__name__)
@@ -133,7 +133,10 @@ def _find_stack_from_landing_path(
 
 
 def _assess_transplant_request(
-    phab: PhabricatorClient, landing_path: list[tuple[int, int]], relman_group_phid: str
+    lando_user: User,
+    phab: PhabricatorClient,
+    landing_path: list[tuple[int, int]],
+    relman_group_phid: str,
 ) -> tuple[
     TransplantAssessment,
     Optional[list[tuple[dict, dict]]],
@@ -158,8 +161,9 @@ def _assess_transplant_request(
     )
 
     assessment = check_landing_blockers(
-        g.auth0_user, landing_path_phid, stack_data, landable, landable_repos
+        lando_user, landing_path_phid, stack_data, landable, landable_repos
     )
+
     if assessment.blocker is not None:
         return (assessment, None, None, None)
 
@@ -195,7 +199,7 @@ def _assess_transplant_request(
 
     assessment = check_landing_warnings(
         phab,
-        g.auth0_user,
+        lando_user,
         to_land,
         repo,
         landing_repo,
@@ -209,24 +213,31 @@ def _assess_transplant_request(
     return (assessment, to_land, landing_repo, stack_data)
 
 
-# TODO: auth stuff
-@auth.require_auth0(scopes=("lando", "profile", "email"), userinfo=True)
 @require_phabricator_api_key(optional=True)
-def dryrun(phab: PhabricatorClient, data: dict):
+def dryrun(phab: PhabricatorClient, request, data: dict):
+    lando_user = request.user
+    if not lando_user.is_authenticated:
+        raise PermissionDenied
+
     landing_path = _parse_transplant_request(data)["landing_path"]
 
     release_managers = get_release_managers(phab)
     if not release_managers:
         raise Exception("Could not find `#release-managers` project on Phabricator.")
 
     relman_group_phid = phab.expect(release_managers, "phid")
-    assessment, *_ = _assess_transplant_request(phab, landing_path, relman_group_phid)
+    assessment, *_ = _assess_transplant_request(
+        lando_user, phab, landing_path, relman_group_phid
+    )
     return assessment.to_dict()
 
 
-@auth.require_auth0(scopes=("lando", "profile", "email"), userinfo=True)
 @require_phabricator_api_key(optional=True)
-def post(phab: PhabricatorClient, data: dict):
+def post(phab: PhabricatorClient, request, data: dict):
+    lando_user = request.user
+    if not lando_user.is_authenticated:
+        raise PermissionDenied
+
     parsed_transplant_request = _parse_transplant_request(data)
     confirmation_token = parsed_transplant_request["confirmation_token"]
     flags = parsed_transplant_request["flags"]
@@ -248,7 +259,7 @@ def post(phab: PhabricatorClient, data: dict):
     relman_group_phid = phab.expect(release_managers, "phid")
 
     assessment, to_land, landing_repo, stack_data = _assess_transplant_request(
-        phab, landing_path, relman_group_phid
+        lando_user, phab, landing_path, relman_group_phid
     )
 
     assessment.raise_if_blocked_or_unacknowledged(confirmation_token)
@@ -353,6 +364,7 @@ def post(phab: PhabricatorClient, data: dict):
         lando_revision = Revision.get_from_revision_id(revision_id)
         if not lando_revision:
             lando_revision = Revision(revision_id=revision_id)
+
         lando_revision.diff_id = diff_id
         lando_revision.save()
 
@@ -369,11 +381,13 @@ def post(phab: PhabricatorClient, data: dict):
         }
 
         raw_diff = phab.call_conduit("differential.getrawdiff", diffID=diff["id"])
-        lando_revision.set_patch(raw_diff, patch_data)
+        lando_revision.set_patch(
+            raw_diff, RepoTypeEnum(landing_repo.repo_type), patch_data
+        )
         lando_revision.save()
         lando_revisions.append(lando_revision)
 
-    ldap_username = g.auth0_user.email
+    ldap_username = lando_user.email
 
     submitted_assessment = TransplantAssessment(
         blocker=(
@@ -397,8 +411,10 @@ def post(phab: PhabricatorClient, data: dict):
             requester_email=ldap_username,
             repository_name=landing_repo.short_name,
             repository_url=landing_repo.url,
+            target_repo=landing_repo,
         )
         job.save()
+
     add_revisions_to_job(lando_revisions, job)
     logger.info(f"Setting {revision_reviewers} reviewer data on each revision.")
     for revision in lando_revisions:
@@ -410,7 +426,7 @@ def post(phab: PhabricatorClient, data: dict):
     job.set_landed_revision_diffs()
     job.save()
 
-    logger.info(f"New landing job {job.id} created for {landing_repo.tree} repo.")
+    logger.info(f"New landing job {job.id} created for {landing_repo.name} repo.")
 
     # Asynchronously remove the checkin project from any of the landing
     # revisions that had it.
@@ -429,7 +445,7 @@ def post(phab: PhabricatorClient, data: dict):
 
 
 @require_phabricator_api_key(optional=True)
-def get_list(phab: PhabricatorClient, stack_revision_id: str):
+def get_list(phab: PhabricatorClient, request, stack_revision_id: str):
     """Return a list of Transplant objects"""
     revision_id_int = revision_id_to_int(stack_revision_id)
 
@@ -456,4 +472,4 @@ def get_list(phab: PhabricatorClient, stack_revision_id: str):
 
     landing_jobs = LandingJob.revisions_query(rev_ids).all()
 
-    return [job.serialize() for job in landing_jobs], 200
+    return [job.serialize() for job in landing_jobs]