From 4de1bf1783493707b4f6ba17d12533e99ea38082 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 25 Aug 2023 16:42:09 +0200 Subject: [PATCH 1/6] Expand functionality of find_modules so that it can be passed a regular expression. It can also ommit the version (i.e. anything after the last slash) from the match, if required. --- eessi/testsuite/utils.py | 59 ++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index 3b770d77..dcec83f3 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -4,7 +4,7 @@ import re import sys -from typing import Iterator +from typing import Iterator, List import reframe as rfm import reframe.core.runtime as rt @@ -57,15 +57,58 @@ def is_cuda_required_module(module_name: str) -> bool: return requires_cuda -def find_modules(substr: str) -> Iterator[str]: - """Return all modules in the current system that contain ``substr`` in their name.""" - if not isinstance(substr, str): +def find_modules(regex: str, name_only = True) -> Iterator[str]: + """ + Return all modules matching the regular expression regex. Note that since we use re.search, + a module matches if the regex matches the module name at any place. I.e. the match does + not have to be at the start of the smodule name + + Arguments: + - regex: a regular expresion + - name_only: regular expressions will only be matched on the module name, not the version (default: True). + + Note: the name_only feature assumes anything after the last forward '/' is the version, + and strips that before doing a match. + + Example + + Suppose we have the following modules on a system: + + gompic/2022a + gompi/2022a + CGAL/4.14.3-gompi-2022a + + The following calls would return the following respective modules + + find_modules('gompi') => [gompic/2022a, gompi/2022a] + find_modules('gompi$') => [gompi/2022a] + find_modules('gompi', name_only = False) => [gompic/2022a, gompi/2022a, CGAL/4.14.3-gompi-2022a] + find_modules('^gompi', name_only = False) => [gompic/2022a, gompi/2022a] + find_modules('^gompi\/', name_only = False) => [gompi/2022a] + find_modules('-gompi-2022a', name_only = False) => [CGAL/4.14.3-gompi-2022a] + + """ + + if not isinstance(regex, str): raise TypeError("'substr' argument must be a string") ms = rt.runtime().modules_system - modules = OrderedSet(ms.available_modules(substr)) - for m in modules: - yield m + # Returns e.g. ['Bison/', 'Bison/3.7.6-GCCcore-10.3.0', 'BLIS/', 'BLIS/0.8.1-GCC-10.3.0'] + modules = ms.available_modules('') + for mod in modules: + # Exclude anything without version, i.e. ending with / (e.g. Bison/) + if re.search('.*\/$', mod): + continue + # The thing we yield should always be the original module name (orig_mod), including version + orig_mod = mod + if name_only: + # Remove part after the last forward slash, as we assume this is the version + mod = re.sub('\/[^\/]*$', '', mod) + # Match the actual regular expression + log(f"Matching module {mod} with regex {regex}") + if re.search(regex, mod): + log(f"Match!") + yield orig_mod def check_proc_attribute_defined(test: rfm.RegressionTest, attribute) -> bool: """ @@ -104,4 +147,4 @@ def check_proc_attribute_defined(test: rfm.RegressionTest, attribute) -> bool: "This is a programming error, please report this issue." ) raise AttributeError(msg) - \ No newline at end of file + From 90b7f08bf187890957fdde9b4b255cbe74cb0cf4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 25 Aug 2023 16:46:45 +0200 Subject: [PATCH 2/6] Importing List from typing isn't needed --- eessi/testsuite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index dcec83f3..ed6c611d 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -4,7 +4,7 @@ import re import sys -from typing import Iterator, List +from typing import Iterator import reframe as rfm import reframe.core.runtime as rt From b639998e05428f4b048a8b338a63ba7a104a8519 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:52:51 +0200 Subject: [PATCH 3/6] Apply suggestions from code review Forward slash does not need to be escaped Co-authored-by: Sam Moors --- eessi/testsuite/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index ed6c611d..a18c19fa 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -97,13 +97,13 @@ def find_modules(regex: str, name_only = True) -> Iterator[str]: modules = ms.available_modules('') for mod in modules: # Exclude anything without version, i.e. ending with / (e.g. Bison/) - if re.search('.*\/$', mod): + if re.search('.*/$', mod): continue # The thing we yield should always be the original module name (orig_mod), including version orig_mod = mod if name_only: # Remove part after the last forward slash, as we assume this is the version - mod = re.sub('\/[^\/]*$', '', mod) + mod = re.sub('/[^/]*$', '', mod) # Match the actual regular expression log(f"Matching module {mod} with regex {regex}") if re.search(regex, mod): From ecb3fc5412c366fe872da5ff99bfb35296157f0b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:09:18 +0200 Subject: [PATCH 4/6] Update eessi/testsuite/utils.py Co-authored-by: Sam Moors --- eessi/testsuite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index a18c19fa..a6fd2478 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -84,7 +84,7 @@ def find_modules(regex: str, name_only = True) -> Iterator[str]: find_modules('gompi$') => [gompi/2022a] find_modules('gompi', name_only = False) => [gompic/2022a, gompi/2022a, CGAL/4.14.3-gompi-2022a] find_modules('^gompi', name_only = False) => [gompic/2022a, gompi/2022a] - find_modules('^gompi\/', name_only = False) => [gompi/2022a] + find_modules('^gompi/', name_only = False) => [gompi/2022a] find_modules('-gompi-2022a', name_only = False) => [CGAL/4.14.3-gompi-2022a] """ From 0071c7b6cbabf7881bc8ff1189fb6d0baaab3f81 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:09:50 +0200 Subject: [PATCH 5/6] Apply style suggestions from code review Co-authored-by: Sam Moors --- eessi/testsuite/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index a6fd2478..e75d7eb6 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -57,14 +57,14 @@ def is_cuda_required_module(module_name: str) -> bool: return requires_cuda -def find_modules(regex: str, name_only = True) -> Iterator[str]: +def find_modules(regex: str, name_only=True) -> Iterator[str]: """ Return all modules matching the regular expression regex. Note that since we use re.search, a module matches if the regex matches the module name at any place. I.e. the match does not have to be at the start of the smodule name Arguments: - - regex: a regular expresion + - regex: a regular expression - name_only: regular expressions will only be matched on the module name, not the version (default: True). Note: the name_only feature assumes anything after the last forward '/' is the version, From 65a25186d622c81ce5d997b6107f078b8f37cf94 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 31 Aug 2023 17:15:42 +0200 Subject: [PATCH 6/6] Added suggested stripping of trailing slashes in case we do matching with name_only=True --- eessi/testsuite/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index ed6c611d..a754c2cf 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -102,6 +102,8 @@ def find_modules(regex: str, name_only = True) -> Iterator[str]: # The thing we yield should always be the original module name (orig_mod), including version orig_mod = mod if name_only: + # Remove trailing slashes from the regex (in case the callee forgot) + regex = regex.rstrip('/') # Remove part after the last forward slash, as we assume this is the version mod = re.sub('\/[^\/]*$', '', mod) # Match the actual regular expression