Cleanup

executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py

@@ -473,9 +473,9 @@ def __init__(self, initial_job_request: V0InitialJobRequest | V1InitialJobReques
         self.specs_volume_mount_dir = self.temp_dir / "specs"
         self.download_manager = DownloadManager()
 
-        self.job_container_name = f"{settings.EXECUTOR_TOKEN}-job"
-        self.nginx_container_name = f"{settings.EXECUTOR_TOKEN}-nginx"
-        self.job_network_name = f"{settings.EXECUTOR_TOKEN}-network"
+        self.job_container_name = f"ch-{settings.EXECUTOR_TOKEN}-job"
+        self.nginx_container_name = f"ch-{settings.EXECUTOR_TOKEN}-nginx"
+        self.job_network_name = f"ch-{settings.EXECUTOR_TOKEN}-network"
         self.process: asyncio.subprocess.Process | None = None
         self.cmd: list[str] = []
 
@@ -489,12 +489,26 @@ def __init__(self, initial_job_request: V0InitialJobRequest | V1InitialJobReques
             save_public_key(self.initial_job_request.public_key, self.nginx_dir_path)
             self.is_streaming_job = True
 
+    async def cleanup_potential_old_jobs(self):
+        await (
+            await asyncio.create_subprocess_shell(
+                "docker kill $(docker ps -q --filter 'name=ch-.*-job')"
+            )
+        ).communicate()
+        await (
+            await asyncio.create_subprocess_shell(
+                "docker kill $(docker ps -q --filter 'name=ch-.*-nginx')"
+            )
+        ).communicate()
+
     async def prepare(self):
         self.volume_mount_dir.mkdir(exist_ok=True)
         self.output_volume_mount_dir.mkdir(exist_ok=True)
 
         logger.info("preparing in progress")
 
+        await self.cleanup_potential_old_jobs()
+
         if self.initial_job_request.base_docker_image_name is not None:
             logger.info("docker pull %s", self.initial_job_request.base_docker_image_name)
             process = await asyncio.create_subprocess_exec(

executor/app/src/compute_horde_executor/executor/tests/integration/test_main_loop.py

@@ -4,6 +4,7 @@
 import logging
 import random
 import string
+import subprocess
 import uuid
 import zipfile
 from functools import partial
@@ -70,6 +71,25 @@ def __init__(self, messages, *args, **kwargs):
 
 
 def test_main_loop():
+    job_container_name = f"ch-{uuid.uuid4()}-job"
+    nginx_container_name = f"ch-{uuid.uuid4()}-nginx"
+    for container_name in [job_container_name, nginx_container_name]:
+        subprocess.check_output(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                container_name,
+                "busybox",
+                "sleep",
+                "1000",
+            ]
+        )
+    for container_name in [job_container_name, nginx_container_name]:
+        output = subprocess.check_output(["docker", "ps", "--filter", f"name={container_name}"])
+        assert container_name.encode() in output
+
     command = CommandTested(
         iter(
             [
@@ -117,6 +137,10 @@ def test_main_loop():
         },
     ]
 
+    for container_name in [job_container_name, nginx_container_name]:
+        output = subprocess.check_output(["docker", "ps", "--filter", f"name={container_name}"])
+        assert container_name.encode() not in output
+
 
 def test_main_loop_streaming_job():
     _, public_key, _ = generate_certificate_at()

executor/noxfile.py

@@ -159,7 +159,7 @@ def test(session):
             "-x",
             "-vv",
             "-n",
-            "auto",
+            "1",
             "--junitxml",
             "test-report.xml",
             "compute_horde_executor",