try to fix silent failures with multi_run script; fix error message a…

…t end of run_gmtb_scm.py script, even when successful; add MG INCCN data to physics_input_data; add more years of co2historicaldata*.txt files

scm/src/gmtb_scm.F90

@@ -147,7 +147,7 @@ subroutine gmtb_scm_main_sub()
   write(0,'(a,i0,a,i0)') "Called ccpp_physics_init with suite '" // trim(trim(adjustl(scm_state%physics_suite_name))) // "', ierr=", ierr
   if (ierr/=0) then
       write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_init: ' // trim(cdata%errmsg) // '. Exiting...'
-      stop
+      stop 1
   end if
 
   physics%Model%first_time_step = .true.
@@ -214,7 +214,7 @@ subroutine gmtb_scm_main_sub()
     call ccpp_physics_run(cdata, suite_name=trim(trim(adjustl(scm_state%physics_suite_name))), ierr=ierr)
     if (ierr/=0) then
         write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_run: ' // trim(cdata%errmsg) // '. Exiting...'
-        stop
+        stop 1
     end if
 
     !the filter routine (called after the following leapfrog time step) expects time level 2 in temp_tracer to be the updated, unfiltered state after the previous time step
@@ -311,7 +311,7 @@ subroutine gmtb_scm_main_sub()
 
   if (ierr/=0) then
       write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_finalize: ' // trim(cdata%errmsg) // '. Exiting...'
-      stop
+      stop 1
   end if
 
 end subroutine gmtb_scm_main_sub

scm/src/multi_run_gmtb_scm.py

@@ -69,7 +69,12 @@ def subprocess_work(command):
         message = '####### The subprocess started using the command ({0}) exited with code {1}. #######\n'\
                   'Run the command ({0}) by itself again or use the -v or -vv options for more details.'.format(command, exit_code)
         logging.critical(message)
-        #raise Exception(message)
+    elif 'SystemExit' in str(output):
+        exit_code = str(output).split("SystemExit: ")[-1].split("\\n")[0].strip()
+        message = '####### The subprocess started using the command ({0}) exited with a normal exit code, but\n'\
+        'the terminal output indicated that an error occurred ({1}). #######\n'\
+        'Run the command ({0}) by itself again or use the -v or -vv options for more details.'.format(command, exit_code)
+        logging.critical(message)
     RESULTS.append([command, exit_code])
 
 def main():

scm/src/run_gmtb_scm.py

@@ -397,7 +397,9 @@ def launch_executable(use_gdb, gdb):
         cmd = '{executable}'.format(executable=EXECUTABLE)
     logging.info('Passing control to "{0}"'.format(cmd))
     time.sleep(2)
-    sys.exit(os.system(cmd))
+    return_code = os.system(cmd)
+    if return_code != 0:
+        sys.exit(return_code)
 
 def copy_outdir(exp_dir):
     """Copy output directory to /home for this experiment."""