From 3d45800fd025a41240baa788b142d2bbfd234abf Mon Sep 17 00:00:00 2001
From: Grant Firl <grantf@ucar.edu>
Date: Fri, 19 Feb 2021 20:29:43 -0700
Subject: [PATCH] try to fix silent failures with multi_run script; fix error
 message at end of run_gmtb_scm.py script, even when successful; add MG INCCN
 data to physics_input_data; add more years of co2historicaldata*.txt files

---
 scm/src/gmtb_scm.F90          | 6 +++---
 scm/src/multi_run_gmtb_scm.py | 7 ++++++-
 scm/src/run_gmtb_scm.py       | 4 +++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/scm/src/gmtb_scm.F90 b/scm/src/gmtb_scm.F90
index 4660e923e..b8f15c106 100644
--- a/scm/src/gmtb_scm.F90
+++ b/scm/src/gmtb_scm.F90
@@ -147,7 +147,7 @@ subroutine gmtb_scm_main_sub()
   write(0,'(a,i0,a,i0)') "Called ccpp_physics_init with suite '" // trim(trim(adjustl(scm_state%physics_suite_name))) // "', ierr=", ierr
   if (ierr/=0) then
       write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_init: ' // trim(cdata%errmsg) // '. Exiting...'
-      stop
+      stop 1
   end if
   
   physics%Model%first_time_step = .true.
@@ -214,7 +214,7 @@ subroutine gmtb_scm_main_sub()
     call ccpp_physics_run(cdata, suite_name=trim(trim(adjustl(scm_state%physics_suite_name))), ierr=ierr)
     if (ierr/=0) then
         write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_run: ' // trim(cdata%errmsg) // '. Exiting...'
-        stop
+        stop 1
     end if
 
     !the filter routine (called after the following leapfrog time step) expects time level 2 in temp_tracer to be the updated, unfiltered state after the previous time step
@@ -311,7 +311,7 @@ subroutine gmtb_scm_main_sub()
 
   if (ierr/=0) then
       write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_finalize: ' // trim(cdata%errmsg) // '. Exiting...'
-      stop
+      stop 1
   end if
 
 end subroutine gmtb_scm_main_sub
diff --git a/scm/src/multi_run_gmtb_scm.py b/scm/src/multi_run_gmtb_scm.py
index 65ce4d740..6d71bbccf 100755
--- a/scm/src/multi_run_gmtb_scm.py
+++ b/scm/src/multi_run_gmtb_scm.py
@@ -69,7 +69,12 @@ def subprocess_work(command):
         message = '####### The subprocess started using the command ({0}) exited with code {1}. #######\n'\
                   'Run the command ({0}) by itself again or use the -v or -vv options for more details.'.format(command, exit_code)
         logging.critical(message)
-        #raise Exception(message)
+    elif 'SystemExit' in str(output):
+        exit_code = str(output).split("SystemExit: ")[-1].split("\\n")[0].strip()
+        message = '####### The subprocess started using the command ({0}) exited with a normal exit code, but\n'\
+        'the terminal output indicated that an error occurred ({1}). #######\n'\
+        'Run the command ({0}) by itself again or use the -v or -vv options for more details.'.format(command, exit_code)
+        logging.critical(message)
     RESULTS.append([command, exit_code])
 
 def main():
diff --git a/scm/src/run_gmtb_scm.py b/scm/src/run_gmtb_scm.py
index 0abc8303d..2e6e4d289 100755
--- a/scm/src/run_gmtb_scm.py
+++ b/scm/src/run_gmtb_scm.py
@@ -397,7 +397,9 @@ def launch_executable(use_gdb, gdb):
         cmd = '{executable}'.format(executable=EXECUTABLE)
     logging.info('Passing control to "{0}"'.format(cmd))
     time.sleep(2)
-    sys.exit(os.system(cmd))
+    return_code = os.system(cmd)
+    if return_code != 0:
+        sys.exit(return_code)
     
 def copy_outdir(exp_dir):
     """Copy output directory to /home for this experiment."""