Merge branch 'NOAA-EMC:develop' into feature/gefs-arch

AntonMFernando-NOAA · Aug 13, 2024 · 710c387 · 710c387
2 parents 9e43bda + 336b78a
commit 710c387
Showing 1 changed file with 11 additions and 4 deletions.
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
@@ -136,7 +136,7 @@ def rocoto_statcount(rocotostat):
     rocotostat_output = [line.split()[0:4] for line in rocotostat_output]
     rocotostat_output = [line for line in rocotostat_output if len(line) != 1]
 
-    status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED']
+    status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE']
 
     rocoto_status = {}
     status_counts = Counter(case for sublist in rocotostat_output for case in sublist)
@@ -214,9 +214,16 @@ def is_stalled(rocoto_status):
     elif rocoto_status['DEAD'] > 0:
         error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
         rocoto_state = 'FAIL'
-    elif 'UNKNOWN' in rocoto_status:
-        error_return = rocoto_status['UNKNOWN']
-        rocoto_state = 'UNKNOWN'
+    elif 'UNAVAILABLE' in rocoto_status or 'UNKNOWN' in rocoto_status:
+        rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError)
+        error_return = 0
+        rocoto_state = 'RUNNING'
+        if 'UNAVAILABLE' in rocoto_status:
+            error_return = rocoto_status['UNAVAILABLE']
+            rocoto_state = 'UNAVAILABLE'
+        if 'UNKNOWN' in rocoto_status:
+            error_return += rocoto_status['UNKNOWN']
+            rocoto_state = 'UNKNOWN'
     elif is_stalled(rocoto_status):
         rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError)
         if is_stalled(rocoto_status):