Add detailed experiment status (#663)

* Add detailed status to experiments Co-authored-by: Davide Eynard <[email protected]> Signed-off-by: javiermtorres <[email protected]>
mozilla-ai · Jan 30, 2025 · a7ad438 · a7ad438
1 parent 57682b2
commit a7ad438
Showing 1 changed file with 19 additions and 1 deletion.
diff --git a/lumigator/python/mzai/backend/backend/services/experiments.py b/lumigator/python/mzai/backend/backend/services/experiments.py
@@ -62,14 +62,32 @@ def get_experiment(self, experiment_id: UUID) -> ExperimentResponse:
         loguru.logger.info(f"Obtaining info for experiment {experiment_id}: {record}")
 
         all_succeeded = True
+        any_running = False
+        any_failed = False
+        # Any running: running
+        # Any failed: failed
+        # All succeeded: succeeded
         for job in self._get_all_owned_jobs(experiment_id):
             loguru.logger.info(f"Checking sub job: {job}")
             if self._job_service.get_job(job.id).status != JobStatus.SUCCEEDED:
                 all_succeeded = False
-                break
+                if self._job_service.get_job(job.id).status == JobStatus.FAILED:
+                    any_failed = True
+                    # Any failed job makes further inspection unnecessary
+                    break
+                if self._job_service.get_job(job.id).status == JobStatus.RUNNING:
+                    any_running = True
+                    # Any running job will move status to running, but
+                    # searching for failures is still necessary
 
         if all_succeeded:
             record = self._experiment_repo.update(experiment_id, status=JobStatus.SUCCEEDED)
+        elif any_failed:
+            record = self._experiment_repo.update(experiment_id, status=JobStatus.FAILED)
+        elif any_running:
+            record = self._experiment_repo.update(experiment_id, status=JobStatus.RUNNING)
+        else:
+            record = self._experiment_repo.update(experiment_id, status=JobStatus.PENDING)
 
         return ExperimentResponse.model_validate(record)