use absolute paths for file logs to directly save in prefix

EESSI · Sep 12, 2023 · b3ea0c3 · b3ea0c3
1 parent 5d5ba7f
commit b3ea0c3
Show file tree

Hide file tree

Showing 8 changed files with 52 additions and 51 deletions.
diff --git a/README.md b/README.md
@@ -37,12 +37,11 @@ cd /path/to/EESSI/test-suite
 module load ReFrame/4.2.0
 
 export PYTHONPATH=$PWD:$PYTHONPATH
+export RFM_PREFIX=<path_to_all_output>
 
 reframe \
     --config-file <path_to_site_config_file> \
     --checkpath eessi/testsuite/tests/apps \
-    --prefix <path_to_all_output> \
-    --save-log-files \
     --tag CI --tag 1_node \
     --run --performance-report
 ```

diff --git a/config/aws_citc.py b/config/aws_citc.py
@@ -16,9 +16,8 @@
 from eessi.testsuite.common_config import common_logging_config
 from eessi.testsuite.constants import FEATURES
 
-# This config will write all staging, output (and logging with --save-log-files) to subdirs under this prefix
-# Override with --prefix
-reframe_prefix = f'{os.environ.get("HOME")}/reframe_runs'
+# This config will write all staging, output and logging to subdirs under this prefix
+reframe_prefix = os.path.join(os.environ['HOME'], 'reframe_runs')
 
 # AWS CITC site configuration
 site_configuration = {
@@ -116,11 +115,10 @@
             'ftn': '',
         },
     ],
-    'logging': common_logging_config,
+    'logging': common_logging_config(reframe_prefix),
     'general': [
         {
             'remote_detect': True,
-            'save_log_files': True,
         }
     ],
 }

diff --git a/config/github_actions.py b/config/github_actions.py
@@ -36,8 +36,7 @@
         {
             'purge_environment': True,
             'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
-            'save_log_files': True,
         }
     ],
-    'logging': common_logging_config,
+    'logging': common_logging_config(),
 }
diff --git a/config/izum_vega.py b/config/izum_vega.py
@@ -3,9 +3,8 @@
 from eessi.testsuite.common_config import common_logging_config
 from eessi.testsuite.constants import *  # noqa: F403
 
-# This config will write all staging, output (and logging with --save-log-files) to subdirs under this prefix
-# Override with --prefix
-reframe_prefix = f'{os.environ.get("HOME")}/reframe_runs'
+# This config will write all staging, output and logging to subdirs under this prefix
+reframe_prefix = os.path.join(os.environ['HOME'], 'reframe_runs')
 
 # This is an example configuration file
 site_configuration = {
@@ -14,7 +13,6 @@
             # Enable automatic detection of CPU architecture for each partition
             # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information
             'remote_detect': True,
-            'save_log_files': True,
         }
     ],
     'systems': [
@@ -94,5 +92,5 @@
             'ftn': '',
         },
     ],
-    'logging': common_logging_config,
+    'logging': common_logging_config(reframe_prefix),
 }
diff --git a/config/settings_example.py b/config/settings_example.py
@@ -78,10 +78,7 @@
             'ftn': '',
         },
     ],
-    'logging': common_logging_config,
-    'general': {
-        'save_log_files': True,  # copy ReFrame file logs to <prefix>/output on exit
-    },
+    'logging': common_logging_config(),
 }
 
 # optional logging to syslog

diff --git a/config/surf_snellius.py b/config/surf_snellius.py
@@ -66,14 +66,13 @@
             'ftn': '',
         },
     ],
-    'logging': common_logging_config,
+    'logging': common_logging_config(),
     'general': [
         {
             # For autodetect to work, temporarily change:
             # 1. The launchers to srun
             # 2. Add --exclusive to GPU 'access' field above (avoids submission error that no GPUs are requested)
             'remote_detect': True,
-            'save_log_files': True,
         }
     ],
 }
diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py
@@ -195,8 +195,7 @@ def command(self, job):
         {
             'purge_environment': True,
             'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
-            'save_log_files': True,
         }
     ],
-    'logging': common_logging_config,
+    'logging': common_logging_config(),
 }
diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py
@@ -1,3 +1,5 @@
+import os
+
 perflog_format = '|'.join([
     '%(check_job_completion_time)s',
     '%(osuser)s',
@@ -27,32 +29,42 @@
     ''  # final delimiter required
 ])
 
-common_logging_config = [{
-    'level': 'debug',
-    'handlers': [
-        {
-            'type': 'stream',
-            'name': 'stdout',
-            'level': 'info',
-            'format': '%(message)s',
-        },
-        {
-            'type': 'file',
-            'name': 'reframe.log',
-            'level': 'debug',
-            'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s',
-            'append': True,
-            'timestamp': "%Y%m%d_%H%M%S",  # add a timestamp to the filename (reframe_<timestamp>.log)
-        },
-    ],
-    'handlers_perflog': [
-        {
-            'type': 'filelog',
-            'prefix': '%(check_system)s/%(check_partition)s',
-            'level': 'info',
-            'format': perflog_format,
-            'format_perfvars': format_perfvars,
-            'append': True,  # avoid overwriting
-        },
-    ],
-}]
+
+def common_logging_config(prefix=None):
+    """
+    return default logging configuration as a list: stdout, file log, perflog
+    :param prefix: file log prefix
+    """
+    prefix = os.getenv('RFM_PREFIX', prefix if prefix else '.')
+
+    os.makedirs(os.path.join(prefix, 'log'), exist_ok=True)
+
+    return [{
+        'level': 'debug',
+        'handlers': [
+            {
+                'type': 'stream',
+                'name': 'stdout',
+                'level': 'info',
+                'format': '%(message)s',
+            },
+            {
+                'type': 'file',
+                'name': os.path.join(prefix, 'log', 'reframe.log'),
+                'level': 'debug',
+                'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s',
+                'append': True,
+                'timestamp': "%Y%m%d_%H%M%S",  # add a timestamp to the filename (reframe_<timestamp>.log)
+            },
+        ],
+        'handlers_perflog': [
+            {
+                'type': 'filelog',
+                'prefix': '%(check_system)s/%(check_partition)s',
+                'level': 'info',
+                'format': perflog_format,
+                'format_perfvars': format_perfvars,
+                'append': True,  # avoid overwriting
+            },
+        ],
+    }]