From b3ea0c32fe1324de12b551ca8d48628082dede09 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 12 Sep 2023 09:13:43 +0200 Subject: [PATCH] use absolute paths for file logs to directly save in prefix --- README.md | 3 +- config/aws_citc.py | 8 ++-- config/github_actions.py | 3 +- config/izum_vega.py | 8 ++-- config/settings_example.py | 5 +-- config/surf_snellius.py | 3 +- config/vsc_hortense.py | 3 +- eessi/testsuite/common_config.py | 70 +++++++++++++++++++------------- 8 files changed, 52 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index dae68a6f..d67095d3 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,11 @@ cd /path/to/EESSI/test-suite module load ReFrame/4.2.0 export PYTHONPATH=$PWD:$PYTHONPATH +export RFM_PREFIX= reframe \ --config-file \ --checkpath eessi/testsuite/tests/apps \ - --prefix \ - --save-log-files \ --tag CI --tag 1_node \ --run --performance-report ``` diff --git a/config/aws_citc.py b/config/aws_citc.py index e2740ac3..c0004e09 100644 --- a/config/aws_citc.py +++ b/config/aws_citc.py @@ -16,9 +16,8 @@ from eessi.testsuite.common_config import common_logging_config from eessi.testsuite.constants import FEATURES -# This config will write all staging, output (and logging with --save-log-files) to subdirs under this prefix -# Override with --prefix -reframe_prefix = f'{os.environ.get("HOME")}/reframe_runs' +# This config will write all staging, output and logging to subdirs under this prefix +reframe_prefix = os.path.join(os.environ['HOME'], 'reframe_runs') # AWS CITC site configuration site_configuration = { @@ -116,11 +115,10 @@ 'ftn': '', }, ], - 'logging': common_logging_config, + 'logging': common_logging_config(reframe_prefix), 'general': [ { 'remote_detect': True, - 'save_log_files': True, } ], } diff --git a/config/github_actions.py b/config/github_actions.py index 15df611f..68c55072 100644 --- a/config/github_actions.py +++ b/config/github_actions.py @@ -36,8 +36,7 @@ { 'purge_environment': True, 'resolve_module_conflicts': False, # avoid loading the module before submitting the job - 'save_log_files': True, } ], - 'logging': common_logging_config, + 'logging': common_logging_config(), } diff --git a/config/izum_vega.py b/config/izum_vega.py index 640ec216..8d34c1ae 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -3,9 +3,8 @@ from eessi.testsuite.common_config import common_logging_config from eessi.testsuite.constants import * # noqa: F403 -# This config will write all staging, output (and logging with --save-log-files) to subdirs under this prefix -# Override with --prefix -reframe_prefix = f'{os.environ.get("HOME")}/reframe_runs' +# This config will write all staging, output and logging to subdirs under this prefix +reframe_prefix = os.path.join(os.environ['HOME'], 'reframe_runs') # This is an example configuration file site_configuration = { @@ -14,7 +13,6 @@ # Enable automatic detection of CPU architecture for each partition # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information 'remote_detect': True, - 'save_log_files': True, } ], 'systems': [ @@ -94,5 +92,5 @@ 'ftn': '', }, ], - 'logging': common_logging_config, + 'logging': common_logging_config(reframe_prefix), } diff --git a/config/settings_example.py b/config/settings_example.py index d280f8e8..af910605 100644 --- a/config/settings_example.py +++ b/config/settings_example.py @@ -78,10 +78,7 @@ 'ftn': '', }, ], - 'logging': common_logging_config, - 'general': { - 'save_log_files': True, # copy ReFrame file logs to /output on exit - }, + 'logging': common_logging_config(), } # optional logging to syslog diff --git a/config/surf_snellius.py b/config/surf_snellius.py index 20322be3..43c53597 100644 --- a/config/surf_snellius.py +++ b/config/surf_snellius.py @@ -66,14 +66,13 @@ 'ftn': '', }, ], - 'logging': common_logging_config, + 'logging': common_logging_config(), 'general': [ { # For autodetect to work, temporarily change: # 1. The launchers to srun # 2. Add --exclusive to GPU 'access' field above (avoids submission error that no GPUs are requested) 'remote_detect': True, - 'save_log_files': True, } ], } diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py index 3aa0af4f..270cd66d 100644 --- a/config/vsc_hortense.py +++ b/config/vsc_hortense.py @@ -195,8 +195,7 @@ def command(self, job): { 'purge_environment': True, 'resolve_module_conflicts': False, # avoid loading the module before submitting the job - 'save_log_files': True, } ], - 'logging': common_logging_config, + 'logging': common_logging_config(), } diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py index 5e95ba9e..5fd2c816 100644 --- a/eessi/testsuite/common_config.py +++ b/eessi/testsuite/common_config.py @@ -1,3 +1,5 @@ +import os + perflog_format = '|'.join([ '%(check_job_completion_time)s', '%(osuser)s', @@ -27,32 +29,42 @@ '' # final delimiter required ]) -common_logging_config = [{ - 'level': 'debug', - 'handlers': [ - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s', - }, - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', - 'append': True, - 'timestamp': "%Y%m%d_%H%M%S", # add a timestamp to the filename (reframe_.log) - }, - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': perflog_format, - 'format_perfvars': format_perfvars, - 'append': True, # avoid overwriting - }, - ], -}] + +def common_logging_config(prefix=None): + """ + return default logging configuration as a list: stdout, file log, perflog + :param prefix: file log prefix + """ + prefix = os.getenv('RFM_PREFIX', prefix if prefix else '.') + + os.makedirs(os.path.join(prefix, 'log'), exist_ok=True) + + return [{ + 'level': 'debug', + 'handlers': [ + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s', + }, + { + 'type': 'file', + 'name': os.path.join(prefix, 'log', 'reframe.log'), + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', + 'append': True, + 'timestamp': "%Y%m%d_%H%M%S", # add a timestamp to the filename (reframe_.log) + }, + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': perflog_format, + 'format_perfvars': format_perfvars, + 'append': True, # avoid overwriting + }, + ], + }]