diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c764f37318..966f2b2768 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,11 +27,6 @@ jobs: python -m pip install coverage coverage run --source=reframe ./test_reframe.py coverage report -m - - name: Upload Coverage to Codecov - if: matrix.python-version == '3.8' - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: true unittest-py36: runs-on: ubuntu-20.04 @@ -86,7 +81,7 @@ jobs: run: | docker run reframe-${{ matrix.modules-version }}:latest - tutorialtest: + eb-spack-howto: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -99,7 +94,7 @@ jobs: - name: Build Image for Tutorial Tests run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - docker build -f ci-scripts/dockerfiles/tutorials.dockerfile -t reframe-tutorials:latest . + docker build -f ci-scripts/dockerfiles/eb-spack-howto.dockerfile -t reframe-tutorials:latest . docker logout - name: Run Tutorial Tests run: | diff --git a/.github/workflows/test-flux.yaml b/.github/workflows/test-flux.yaml index b6f01a48d4..13520b4119 100644 --- a/.github/workflows/test-flux.yaml +++ b/.github/workflows/test-flux.yaml @@ -38,6 +38,6 @@ jobs: run: | export PATH=$PWD/bin:$PATH which reframe - flux start reframe -c tutorials/flux -C tutorials/flux/settings.py -l - flux start reframe -c tutorials/flux -C tutorials/flux/settings.py --run - flux start python3 ./test_reframe.py --rfm-user-config=tutorials/flux/settings.py -vvvv + flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l + flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -r + flux start python3 ./test_reframe.py --rfm-user-config=examples/howto/flux/settings.py -vvvv diff --git a/ci-scripts/dockerfiles/tutorials.dockerfile b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile similarity index 80% rename from ci-scripts/dockerfiles/tutorials.dockerfile rename to ci-scripts/dockerfiles/eb-spack-howto.dockerfile index 1bce7487a6..90e4aeb758 100644 --- a/ci-scripts/dockerfiles/tutorials.dockerfile +++ b/ci-scripts/dockerfiles/eb-spack-howto.dockerfile @@ -37,4 +37,4 @@ RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/rfmuser/spack/share/spa ENV BASH_ENV /home/rfmuser/setup.sh -CMD ["/bin/bash", "-c", "./bin/reframe -r -C tutorials/config/lmodsys.py -R -c tutorials/build_systems"] +CMD ["/bin/bash", "-c", "./bin/reframe --system=tutorialsys -r -C examples/tutorial/config/baseline_modules.py -R -c examples/tutorial/easybuild/eb_test.py -c examples/tutorial/spack/spack_test.py"] diff --git a/docs/_static/img/reframe-system-arch.svg b/docs/_static/img/reframe-system-arch.svg new file mode 100644 index 0000000000..4bb51a5bac --- /dev/null +++ b/docs/_static/img/reframe-system-arch.svg @@ -0,0 +1,3 @@ + + +
Entry node
launches
launches
Job Scheduler
parallel launcher
parallel launcher
(local, srun, mpirun, etc.)
reframe -r
Submits
tests
launches
Partition 1
Partition 2
Test job
Test job
Test
job
Partition 3
System
Local,
Slurm,
PBS,
Torque,
etc.
Compute Nodes
 \ No newline at end of file diff --git a/docs/_static/img/test-naming.svg b/docs/_static/img/test-naming.svg new file mode 100644 index 0000000000..9934cfb73b --- /dev/null +++ b/docs/_static/img/test-naming.svg @@ -0,0 +1,3 @@ + + +
stream_test %num_threads=2 %thread_placement=close %stream_binary.elem_type=float /22735d1d @pseudo-cluster:login+clang
  ^build_stream %elem_type=float ~pseudo-cluster:login+clang 'stream_binary /d0622327 @pseudo-cluster:login+clang
stream_test %num_threads=2 %thread_placement=close %stream_binary.elem_type=float /22735d1d @pseudo-cluster:login+clang...
Dependencies or fixtures
Dependencies or...
Test name
Test name
Parameters
Parameters
Fixture scope
Fixture scope
Fixture handle
Fixture handle
Test hash
Test hash
Partition/Environment
Partition/Environme... \ No newline at end of file diff --git a/docs/config_reference.rst b/docs/config_reference.rst index cbe7c8ab26..47f66a013a 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -2,7 +2,7 @@ Configuration Reference *********************** -ReFrame's behavior can be configured through its configuration file (see :doc:`configure`), environment variables and command-line options. +ReFrame's behavior can be configured through its configuration file, environment variables and command-line options. An option can be specified via multiple paths (e.g., a configuration file parameter and an environment variable), in which case command-line options precede environment variables, which in turn precede configuration file options. This section provides a complete reference guide of the configuration options of ReFrame that can be set in its configuration file or specified using environment variables. @@ -115,7 +115,7 @@ System Configuration :required: Yes A list of hostname regular expression patterns in Python `syntax `__, which will be used by the framework in order to automatically select a system configuration. - For the auto-selection process, see `here `__. + For the auto-selection process, check the configuration of the :attr:`~config.autodetect_methods` option. .. py:attribute:: systems.max_local_jobs @@ -244,6 +244,8 @@ System Configuration This option is broken in 4.0. +.. _system-partition-configuration: + System Partition Configuration ============================== @@ -484,7 +486,7 @@ System Partition Configuration .. versionadded:: 4.0.0 ReFrame also allows you to register your own custom launchers simply by defining them in the configuration. - You can follow a small tutorial `here `__. + You can follow a small tutorial :ref:`here `. .. py:attribute:: systems.partitions.access @@ -564,7 +566,7 @@ System Partition Configuration :default: ``8`` The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. - This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. + This option is relevant only when ReFrame executes with the :ref:`asynchronous execution policy `. .. py:attribute:: systems.partitions.prepare_cmds @@ -582,7 +584,7 @@ System Partition Configuration :required: No :default: ``[]`` - A list of job scheduler `resource specification `__ objects. + A list of job scheduler `resource specification <#custom-job-scheduler-resources>`__ objects. .. py:attribute:: systems.partitions.processor @@ -591,7 +593,24 @@ System Partition Configuration :default: ``{}`` Processor information for this partition stored in a `processor info object <#processor-info>`__. - If not set, ReFrame will try to auto-detect this information (see :ref:`proc-autodetection` for more information). + If not set, ReFrame will try to determine this information as follows: + + #. If the processor configuration metadata file in ``~/.reframe/topology/{system}-{part}/processor.json`` exists, the topology information is loaded from there. + These files are generated automatically by ReFrame from previous runs. + + #. If the corresponding metadata files are not found, the processor information will be auto-detected. + If the system partition is local (i.e., ``local`` scheduler + ``local`` launcher), the processor information is auto-detected unconditionally and stored in the corresponding metadata file for this partition. + If the partition is remote, ReFrame will not try to auto-detect it unless the :envvar:`RFM_REMOTE_DETECT` or the :attr:`general.remote_detect` configuration option is set. + In that case, the steps to auto-detect the remote processor information are the following: + + a. ReFrame creates a fresh clone of itself in a temporary directory created under ``.`` by default. + This temporary directory prefix can be changed by setting the :envvar:`RFM_REMOTE_WORKDIR` environment variable. + b. ReFrame changes to that directory and launches a job that will first bootstrap the fresh clone and then run that clone with ``{launcher} ./bin/reframe --detect-host-topology=topo.json``. + The :option:`--detect-host-topology` option causes ReFrame to detect the topology of the current host, + which in this case would be one of the remote compute nodes. + + In case of errors during auto-detection, ReFrame will simply issue a warning and continue. + .. versionadded:: 3.5.0 @@ -703,7 +722,7 @@ ReFrame can launch containerized applications, but you need to configure properl Custom Job Scheduler Resources ============================== -ReFrame allows you to define custom scheduler resources for each partition that you can then transparently access through the :attr:`~reframe.core.pipeline.RegressionTest.extra_resources` attribute of a regression test. +ReFrame allows you to define custom scheduler resources for each partition that can then be transparently accessed through the :attr:`~reframe.core.pipeline.RegressionTest.extra_resources` attribute of a test or from an environment. .. py:attribute:: systems.partitions.resources.name @@ -967,6 +986,18 @@ They are associated with `system partitions <#system-partition-configuration>`__ It first looks for definitions for the current partition, then for the containing system and, finally, for global definitions (the ``*`` pseudo-system). +.. py:attribute:: environments.resources + + :required: No + :default: ``{}`` + + Scheduler resources associated with this environments. + + This is the equivalent of a test's :attr:`~reframe.core.pipeline.RegressionTest.extra_resources`. + + .. versionadded:: 4.6 + + .. _logging-config-reference: Logging Configuration @@ -1097,6 +1128,7 @@ All logging handlers share the following set of common attributes: ``%(check_executable)s``, The value of the :attr:`~reframe.core.pipeline.RegressionTest.executable` attribute. ``%(check_executable_opts)s``, The value of the :attr:`~reframe.core.pipeline.RegressionTest.executable_opts` attribute. ``%(check_extra_resources)s``, The value of the :attr:`~reframe.core.pipeline.RegressionTest.extra_resources` attribute. + ``%(check_fail_reason)s``, The failure reason if the test has failed. ``%(check_hashcode)s``, The unique hash associated with this test. ``%(check_info)s``, Various information about this test; essentially the return value of the test's :func:`~reframe.core.pipeline.RegressionTest.info` function. ``%(check_job_completion_time)s``, Same as the ``(check_job_completion_time_unix)s`` but formatted according to ``datefmt``. @@ -1169,6 +1201,9 @@ All logging handlers share the following set of common attributes: .. versionadded:: 4.3 The ``%(check_#ALL)s`` special specifier is added. +.. versionadded:: 4.7 + The ``%(check_fail_reason)s`` specifier is added. + .. py:attribute:: logging.handlers.format_perfvars .. py:attribute:: logging.handlers_perflog.format_perfvars @@ -1216,6 +1251,8 @@ All logging handlers share the following set of common attributes: In addition to the format directives supported by the standard library's `time.strftime() `__ function, ReFrame allows you to use the ``%:z`` directive -- a GNU ``date`` extension -- that will print the time zone difference in a RFC3339 compliant way, i.e., ``+/-HH:MM`` instead of ``+/-HHMM``. +.. _file-handler: + The ``file`` log handler ------------------------ @@ -1617,14 +1654,6 @@ General Configuration .. versionadded:: 3.12.0 -.. py:attribute:: general.git_timeout - - :required: No - :default: 5 - - Timeout value in seconds used when checking if a git repository exists. - - .. py:attribute:: general.dump_pipeline_progress Dump pipeline progress for the asynchronous execution policy in ``pipeline-progress.json``. @@ -1636,6 +1665,24 @@ General Configuration .. versionadded:: 3.10.0 +.. py:attribute:: general.flex_alloc_strict + + :required: No + :default: ``False`` + + Fail flexible tests if their minimum task requirement is not satisfied. + + .. versionadded:: 4.7 + + +.. py:attribute:: general.git_timeout + + :required: No + :default: 5 + + Timeout value in seconds used when checking if a git repository exists. + + .. py:attribute:: general.pipeline_timeout Timeout in seconds for advancing the pipeline in the asynchronous execution policy. @@ -1672,7 +1719,6 @@ General Configuration Try to auto-detect processor information of remote partitions as well. This may slow down the initialization of the framework, since it involves submitting auto-detection jobs to the remote partitions. - For more information on how ReFrame auto-detects processor information, you may refer to :ref:`proc-autodetection`. .. versionadded:: 3.7.0 diff --git a/docs/configure.rst b/docs/configure.rst deleted file mode 100644 index 9c2b96376d..0000000000 --- a/docs/configure.rst +++ /dev/null @@ -1,515 +0,0 @@ -================================= -Configuring ReFrame for Your Site -================================= - -ReFrame comes pre-configured with a minimal generic configuration that will allow you to run ReFrame on any system. -This will allow you to run simple local tests using the default compiler of the system. -Of course, ReFrame is much more powerful than that. -This section will guide you through configuring ReFrame for your site. - -ReFrame's configuration can be either in JSON or in Python format and can be split into multiple files. -The Python format is useful in cases that you want to generate configuration parameters on-the-fly, since ReFrame will import that Python file and the load the resulting configuration. -In the following we will use a single Python-based configuration file also for historical reasons, since it was the only way to configure ReFrame in versions prior to 3.0. - - -.. versionchanged:: 4.0.0 - The configuration can now be split into multiple files. - - -Loading the configuration -------------------------- - -ReFrame builds its final configuration gradually by combining multiple configuration files. -Each one can have different parts of the configuration, for example different systems, different environments, different general options or different logging handlers. -This technique allows users to avoid having a single huge configuration file. - -The first configuration file loaded in this chain is always the generic builtin configuration located under ``${RFM_INSTALL_PREFIX}/reframe/core/settings.py``. -This contains everything that ReFrame needs to run on a generic system, as well as basic settings for logging, so subsequent configuration files may skip defining some configuration sections altogether, if they are not relevant. - -ReFrame continues on looking for configuration files in the directories defined in :envvar:`RFM_CONFIG_PATH`. -For each directory, will look within it for a ``settings.py`` or ``settings.json`` file (in that order), and if it finds one, it will load it. - -Finally, ReFrame processes the :option:`--config-file` option or the :envvar:`RFM_CONFIG_FILES` environment variable to load any specific configuration files passed from the command line. - - -Anatomy of the Configuration File ---------------------------------- - -The whole configuration of ReFrame is a single JSON object whose properties are responsible for configuring the basic aspects of the framework. -We'll refer to these top-level properties as *sections*. -These sections contain other objects which further define in detail the framework's behavior. -If you are using a Python file to configure ReFrame, this big JSON configuration object is stored in a special variable called ``site_configuration``. - -We will explore the basic configuration of ReFrame by looking into the configuration file of the tutorials, which permits ReFrame to run on the Piz Daint supercomputer and a local computer. -For the complete listing and description of all configuration options, you should refer to the :doc:`config_reference`. - -.. literalinclude:: ../tutorials/config/daint.py - :start-at: site_configuration - -There are three required sections that the final ReFrame configuration must have: ``systems``, ``environments`` and ``logging``, but in most cases you will define only the first two, as ReFrame's builtin configuration already defines a reasonable logging configuration. We will first cover these sections and then move on to the optional ones. - -.. tip:: - - These configuration sections may not all be defined in the same configuration file, but can reside in any configuration file that is being loaded. - This is the case of the example configuration shown above, where the ``logging`` section is "missing" as it's defined in ReFrame's builtin configuration. - ---------------------- -Systems Configuration ---------------------- - -ReFrame allows you to configure multiple systems in the same configuration file. -Each system is a different object inside the ``systems`` section. -In our example we define only Piz Daint: - -.. literalinclude:: ../tutorials/config/daint.py - :start-at: 'systems' - :end-before: 'environments' - -Each system is associated with a set of properties, which in this case are the following: - -* ``name``: The name of the system. - This should be an alphanumeric string (dashes ``-`` are allowed) and it will be used to refer to this system in other contexts. -* ``descr``: A detailed description of the system. -* ``hostnames``: This is a list of hostname patterns following the `Python Regular Expression Syntax `__, which will be used by ReFrame when it tries to automatically select a configuration entry for the current system. -* ``modules_system``: This refers to the modules management backend which should be used for loading environment modules on this system. - Multiple backends are supported, as well as the special ``nomod`` backend which implements the different modules system operations as no-ops. - For the complete list of the supported modules systems, see `here `__. -* ``partitions``: The list of partitions that are defined for this system. - Each partition is defined as a separate object. - We devote the rest of this section in system partitions, since they are an essential part of ReFrame's configuration. - -A system partition in ReFrame is not bound to a real scheduler partition. -It is a virtual partition or separation of the system. -In the example shown here, we define three partitions that none of them corresponds to a scheduler partition. -The ``login`` partition refers to the login nodes of the system, whereas the ``gpu`` and ``mc`` partitions refer to two different set of nodes in the same cluster that are effectively separated using Slurm constraints. -Let's pick the ``gpu`` partition and look into it in more detail: - -.. literalinclude:: ../tutorials/config/daint.py - :start-at: 'name': 'gpu' - :end-at: 'max_jobs' - -The basic properties of a partition are the following: - -* ``name``: The name of the partition. - This should be an alphanumeric string (dashes ``-`` are allowed) and it will be used to refer to this partition in other contexts. -* ``descr``: A detailed description of the system partition. -* ``scheduler``: The workload manager (job scheduler) used in this partition for launching parallel jobs. - In this particular example, the `Slurm `__ scheduler is used. - For a complete list of the supported job schedulers, see `here `__. -* ``launcher``: The parallel job launcher used in this partition. - In this case, the ``srun`` command will be used. - For a complete list of the supported parallel job launchers, see `here `__. -* ``access``: A list of scheduler options that will be passed to the generated job script for gaining access to that logical partition. - Notice how in this case, the nodes are selected through a constraint and not an actual scheduler partition. -* ``environs``: The list of environments that ReFrame will use to run regression tests on this partition. - These are just symbolic names that refer to environments defined in the ``environments`` section described below. -* ``max_jobs``: The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. - This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. - - For more partition configuration options, have a look `here `__. - - --------------------------- -Environments Configuration --------------------------- - -We have seen already environments to be referred to by the ``environs`` property of a partition. -An environment in ReFrame is simply a collection of environment modules, environment variables and compiler and compiler flags definitions. -None of these attributes is required. -An environment can simply be empty, in which case it refers to the actual environment that ReFrame runs in. -In fact, this is what the generic fallback configuration of ReFrame does. - -Environments in ReFrame are configured under the ``environments`` section of the documentation. -For each environment referenced inside a partition, a definition of it must be present in this section. -In our example, we define environments for all the basic compilers as well as a default built-in one, which is used with the generic system configuration. -In certain contexts, it is useful to see a ReFrame environment as a wrapper of a programming toolchain (MPI + compiler combination): - -.. literalinclude:: ../tutorials/config/daint.py - :start-at: 'environments' - :end-at: # end of environments - -Each environment is associated with a name. -This name will be used to reference this environment in different contexts, as for example in the ``environs`` property of the system partitions. -A programming environment in ReFrame is essentially a collection of environment modules, environment variables and compiler definitions. - -An important feature in ReFrame's configuration is that you can scope the definition of section objects to different systems or system/partition combinations by using the ``target_systems`` property. -In our example, this means that the ``gnu`` environment will be defined this way only for tests running on the system ``daint``. - - ---------------------- -Logging configuration ---------------------- - -ReFrame has a powerful logging mechanism that gives fine grained control over what information is being logged, where it is being logged and how this information is formatted. -Additionally, it allows for logging performance data from performance tests into different channels. -Let's see how logging is defined in the builtin configuration: - -.. literalinclude:: ../reframe/core/settings.py - :start-at: 'logging' - :end-at: # end of logging - -Logging is configured under the ``logging`` section of the configuration, which is a list of logger objects. -Unless you want to configure logging differently for different systems, a single logger object is enough. -Each logger object is associated with a `logging level `__ stored in the ``level`` property and has a set of logging handlers that are actually responsible for handling the actual logging records. -ReFrame's output is performed through its logging mechanism and that's why there is the special ``handlers$`` property. -The handler defined in this property, in the builtin configuration shown here, defines how exactly the output of ReFrame will be printed. -You will not have to override this in your configuration files, unless you really need to change how ReFrame's output look like. - -As a user you might need to override the ``handlers`` property to define different sinks for ReFrame logs and/or output using different verbosity levels. -Note that you can use multiple handlers at the same time. -All handler objects share a set of common properties. -These are the following: - -* ``type``: This is the type of the handler, which determines its functionality. - Depending on the handler type, handler-specific properties may be allowed or required. - For a complete list of available log handler types, see `here `__. -* ``level``: The cut-off level for messages reaching this handler. - Any message with a lower level number will be filtered out. -* ``format``: A format string for formatting the emitted log record. - ReFrame uses the format specifiers from `Python Logging `__, but also defines its owns specifiers. -* ``datefmt``: A time format string for formatting timestamps. - There are two log record fields that are considered timestamps: (a) ``asctime`` and (b) ``check_job_completion_time``. - ReFrame follows the time formatting syntax of Python's `time.strftime() `__ with a small tweak allowing full RFC3339 compliance when formatting time zone differences. - -We will not go into the details of the individual handlers here. -In this particular example we use three handlers of two distinct types: - -1. A file handler to print debug messages in the ``reframe.log`` file using a more extensive message format that contains a timestamp, the level name etc. -2. A stream handler to print any informational messages (and warnings and errors) from ReFrame to the standard output. - This handles essentially the actual output of ReFrame. -3. A file handler to print the framework's output in the ``reframe.out`` file. - -It might initially seem confusing the fact that there are two ``level`` properties: one at the logger level and one at the handler level. -Logging in ReFrame works hierarchically. -When a message is logged, a log record is created, which contains metadata about the message being logged (log level, timestamp, ReFrame runtime information etc.). -This log record first goes into ReFrame's internal logger, where the record's level is checked against the logger's level (here ``debug``). -If the log record's level exceeds the log level threshold from the logger, it is forwarded to the logger's handlers. -Then each handler filters the log record differently and takes care of formatting the log record's message appropriately. -You can view logger's log level as a general cut off. -For example, if we have set it to ``warning``, no debug or informational messages would ever be printed. - -Finally, there is a special set of handlers for handling performance log messages. -Performance log messages are generated *only* for `performance tests `__, i.e., tests defining the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` or the :attr:`~reframe.core.pipeline.RegressionTest.perf_patterns` attributes. -The performance log handlers are stored in the ``handlers_perflog`` property. -The ``filelog`` handler used in this example will create a file per test and per system/partition combination (``.///.log``) and will append to it the obtained performance data every time a performance test is run. -Notice how the message to be logged is structured in the ``format`` and ``format_perfvars`` properties, such that it can be easily parsed from post processing tools. -Apart from file logging, ReFrame offers more advanced performance logging capabilities through Syslog, Graylog and HTTP. - -For a complete reference of logging configuration parameters, please refer to the :doc:`config_reference`. - - ------------------------------ -General configuration options ------------------------------ - -General configuration options of the framework go under the ``general`` section of the configuration file. -This section is optional and, in fact, we do not define it for our tutorial configuration file. -However, there are several options that can go into this section, but the reader is referred to the :doc:`config_reference` for the complete list. - - ---------------------------- -Other configuration options ---------------------------- - -There is finally one additional optional configuration section that is not discussed here: - -The ``modes`` section defines different execution modes for the framework. -Execution modes are discussed in the :doc:`pipeline` page. - - -.. _building-the-final-config: - -Building the Final Configuration --------------------------------- - -.. versionadded:: 4.0.0 - -As mentioned above ReFrame can build its final configuration incrementally from a series of user-specified configuration files starting from the basic builtin configuration. -We discussed briefly at the beginning of this page how ReFrame locates and loads these configuration files and the documentation of the :option:`-C` option provides more detailed information. -But how are these configuration files actually combined? -This is what we will discuss in this section. - -Configuration objects in the top-level configuration sections can be split in two categories: *named* and *unnamed*. -Named objects are the systems, the environments and the modes and the rest are unnamed. -The named object have a ``name`` property. -When ReFrame builds its final configuration, named objects from newer configuration files are either appended or prepended in their respective sections, but unnamed objects are merged based on their ``target_systems``. -More specifically, new systems are *prepended* in the list of the already defined, whereas environments and modes are *appended*. -The reason for that is that systems are tried from the beginning of the list until a match is found. -See :ref:`pick-system-config` for more information on how ReFrame picks the right system. -If a system is redefined, ReFrame will warn about it, but it will still use the new definition. -This is done for backward compatibility with the old configuration mechanism, where users had to redefine also the builtin systems and environments in their configuration. -Similarly, if an environment or a mode is redefined, ReFrame will issue a warning, but only if the redefinition is at the same scope as the conflicting one. -Again this is done for backward compatibility. - -Given the Piz Daint configuration shown in this section and the ReFrame's builtin configuration, ReFrame will build internally the following configuration: - -.. code-block:: python - - site_configuration = { - 'systems': [ - { - # from the Daint config - 'name': 'daint', - ... - }, - { - # from the builtin config - 'name': 'generic', - ... - } - ], - 'environments': [ - { - # from the builtin config - 'name': 'builtin' - ... - }, - { - # from the Daint config - 'name': 'gnu', - ... - } - ], - 'logging': [ - # from the builtin config - ] - } - -You might wonder why would I need to define multiple objects in sections such as ``logging`` or ``general``. -As mentioned above, ReFrame merges them if they refer to the same target systems, but if they don't they can serve as scopes for the configuration parameters they define. -Imagine the following ``general`` section: - -.. code-block:: python - - 'general': [ - { - 'git_timeout': 5 - }, - { - 'git_timeout': 10, - 'target_systems': ['daint'] - }, - { - 'git_timeout': 20, - 'target_systems': ['tresa'] - } - ] - -This means that the default value for ``git_timeout`` is 5 seconds for any system, but it is 10 for ``daint`` and 20 for ``tresa``. -The nice thing is that you can spread that in multiple configuration files and ReFrame will combine them internally in a single one with the various configuration options indexed by their scope. - - -.. _pick-system-config: - -Picking the Right System Configuration --------------------------------------- - -As discussed previously, ReFrame's configuration file can store the configurations for multiple systems. -When launched, ReFrame will pick the first matching configuration and load it. - -ReFrame uses an auto-detection mechanism to get information about the host it is running on and uses that information to pick the right system configuration. -The default auto-detection method uses the ``hostname`` command, but you can define more methods by setting either the :attr:`autodetect_methods` configuration parameter or the :envvar:`RFM_AUTODETECT_METHODS` environment variable. -After having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match it against the :attr:`~config.systems.hostnames` patterns defined for each system. -The first system whose :attr:`~config.systems.hostnames` match will become the current system and its configuration will be loaded. - -As soon as a system configuration is selected, all configuration objects that have a ``target_systems`` property are resolved against the selected system, and any configuration object that is not applicable is dropped. -So, internally, ReFrame keeps an *instantiation* of the site configuration for the selected system only. -To better understand this, let's assume that we have the following ``environments`` defined: - -.. code:: python - - 'environments': [ - { - 'name': 'cray', - 'modules': ['cray'] - }, - { - 'name': 'gnu', - 'modules': ['gnu'] - }, - { - 'name': 'gnu', - 'modules': ['gnu', 'openmpi'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpif90', - 'target_systems': ['foo'] - } - ], - - -If the selected system is ``foo``, then ReFrame will use the second definition of ``gnu`` which is specific to the ``foo`` system. - -You can override completely the system auto-selection process by specifying a system or system/partition combination with the ``--system`` option, e.g., ``--system=daint`` or ``--system=daint:gpu``. - - -Querying Configuration Options ------------------------------- - -ReFrame offers the powerful ``--show-config`` command-line option that allows you to query any configuration parameter of the framework and see how it is set for the selected system. -Using no arguments or passing ``all`` to this option, the whole configuration for the currently selected system will be printed in JSON format, which you can then pipe to a JSON command line editor, such as `jq `__, and either get a colored output or even generate a completely new ReFrame configuration! - -Passing specific configuration keys in this option, you can query specific parts of the configuration. -Let's see some concrete examples: - -* Query the current system's partitions: - - .. code-block:: console - - ./bin/reframe -C tutorials/config/settings.py --system=daint --show-config=systems/0/partitions - - .. code:: javascript - - [ - { - "name": "login", - "descr": "Login nodes", - "scheduler": "local", - "launcher": "local", - "environs": [ - "gnu", - "intel", - "nvidia", - "cray" - ], - "max_jobs": 10 - }, - { - "name": "gpu", - "descr": "Hybrid nodes", - "scheduler": "slurm", - "launcher": "srun", - "access": [ - "-C gpu", - "-A csstaff" - ], - "environs": [ - "gnu", - "intel", - "nvidia", - "cray" - ], - "max_jobs": 100 - }, - { - "name": "mc", - "descr": "Multicore nodes", - "scheduler": "slurm", - "launcher": "srun", - "access": [ - "-C mc", - "-A csstaff" - ], - "environs": [ - "gnu", - "intel", - "nvidia", - "cray" - ], - "max_jobs": 100 - } - ] - - Check how the output changes if we explicitly set system to ``daint:login``: - - .. code-block:: console - - ./bin/reframe -C tutorials/config/settings.py --system=daint:login --show-config=systems/0/partitions - - - .. code:: javascript - - [ - { - "name": "login", - "descr": "Login nodes", - "scheduler": "local", - "launcher": "local", - "environs": [ - "gnu", - "intel", - "nvidia", - "cray" - ], - "max_jobs": 10 - } - ] - - - ReFrame will internally represent system ``daint`` as having a single partition only. - Notice also how you can use indexes to objects elements inside a list. - -* Query an environment configuration: - - .. code-block:: console - - ./bin/reframe -C tutorials/config/settings.py --system=daint --show-config=environments/@gnu - - .. code:: javascript - - { - "name": "gnu", - "modules": [ - "PrgEnv-gnu" - ], - "cc": "cc", - "cxx": "CC", - "ftn": "ftn", - "target_systems": [ - "daint" - ] - } - - If an object has a ``name`` property you can address it by name using the ``@name`` syntax, instead of its index. - -* Query an environment's compiler: - - .. code-block:: console - - ./bin/reframe -C tutorials/config/settings.py --system=daint --show-config=environments/@gnu/cxx - - .. code:: javascript - - "CC" - - If you explicitly query a configuration value which is not defined in the configuration file, ReFrame will print its default value. - - -.. _proc-autodetection: - -Auto-detecting processor information ------------------------------------- - -.. versionadded:: 3.7.0 - -.. |devices| replace:: :attr:`devices` -.. _devices: config_reference.html#.systems[].partitions[].devices -.. |processor| replace:: :attr:`processor` -.. _processor: config_reference.html#.systems[].partitions[].processor -.. |detect_remote_system_topology| replace:: :attr:`remote_detect` -.. _detect_remote_system_topology: config_reference.html#.general[].remote_detect - -ReFrame is able to detect the processor topology of both local and remote partitions automatically. -The processor and device information are made available to the tests through the corresponding attributes of the :attr:`~reframe.core.pipeline.RegressionTest.current_partition` allowing a test to modify its behavior accordingly. -Currently, ReFrame supports auto-detection of the local or remote processor information only. -It does not support auto-detection of devices, in which cases users should explicitly specify this information using the |devices|_ configuration option. -The processor information auto-detection works as follows: - -#. If the |processor|_ configuration option is defined, then no auto-detection is attempted. - -#. If the |processor|_ configuration option is not defined, ReFrame will look for a processor configuration metadata file in ``~/.reframe/topology/{system}-{part}/processor.json``. - If the file is found, the topology information is loaded from there. - These files are generated automatically by ReFrame from previous runs. - -#. If the corresponding metadata files are not found, the processor information will be auto-detected. - If the system partition is local (i.e., ``local`` scheduler + ``local`` launcher), the processor information is auto-detected unconditionally and stored in the corresponding metadata file for this partition. - If the partition is remote, ReFrame will not try to auto-detect it unless the :envvar:`RFM_REMOTE_DETECT` or the |detect_remote_system_topology|_ configuration option is set. - In that case, the steps to auto-detect the remote processor information are the following: - - a. ReFrame creates a fresh clone of itself in a temporary directory created under ``.`` by default. - This temporary directory prefix can be changed by setting the :envvar:`RFM_REMOTE_WORKDIR` environment variable. - b. ReFrame changes to that directory and launches a job that will first bootstrap the fresh clone and then run that clone with ``{launcher} ./bin/reframe --detect-host-topology=topo.json``. - The :option:`--detect-host-topology` option causes ReFrame to detect the topology of the current host, - which in this case would be the remote compute nodes. - - In case of errors during auto-detection, ReFrame will simply issue a warning and continue. diff --git a/docs/deferrables.rst b/docs/deferrables.rst index be78c33265..f3e5f54b5d 100644 --- a/docs/deferrables.rst +++ b/docs/deferrables.rst @@ -3,7 +3,7 @@ Understanding the Mechanism of Deferrable Functions =================================================== This section describes the mechanism behind deferrable functions, which in ReFrame, they are used for sanity and performance checking. -Generally, writing a new sanity function in a :class:`~reframe.core.pipeline.RegressionTest` is as straightforward as decorating a simple member function with the built-in :func:`~reframe.core.pipeline.RegressionMixin.sanity_function` decorator. +Generally, writing a new sanity function in a :class:`~reframe.core.pipeline.RegressionTest` is as straightforward as decorating a simple member function with the built-in :func:`~reframe.core.builtins.sanity_function` decorator. Behind the scenes, this decorator will convert the Python function into a deferrable function and schedule its evaluation for the sanity stage of the test. However, when dealing with more complex scenarios such as a deferrable function taking as an argument the results from other deferrable functions, it is crucial to understand how a deferrable function differs from a regular Python function, and when is it actually evaluated. @@ -181,7 +181,7 @@ There are some exceptions to this rule, though. The logical :keyword:`and`, :keyword:`or` and :keyword:`not` operators as well as the :keyword:`in` operator cannot be deferred automatically. These operators try to take the truthy value of their arguments by calling :func:`bool ` on them. As we shall see later, applying the :func:`bool ` function on a deferred expression causes its immediate evaluation and returns the result. -If you want to defer the execution of such operators, you should use the corresponding :func:`and_ `, :func:`or_ `, :func:`not_ ` and :func:`contains ` functions in :mod:`reframe.utility.sanity`, which basically wrap the expression in a deferrable function. +If you want to defer the execution of such operators, you should use the corresponding :func:`~reframe.utility.sanity.and_`, :func:`~reframe.utility.sanity.or_`, :func:`~reframe.utility.sanity.not_` and :func:`~reframe.utility.sanity.contains` functions in :mod:`reframe.utility.sanity`, which basically wrap the expression in a deferrable function. In summary deferrable functions have the following characteristics: diff --git a/docs/dependencies.rst b/docs/dependencies.rst index f1b43bc9e5..9ff2634274 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -2,7 +2,7 @@ How Test Dependencies Work In ReFrame ===================================== -Dependencies in ReFrame are defined at the test level using the :func:`depends_on` function, but are projected to the `test cases `__ space. +Dependencies in ReFrame are defined at the test level using the :func:`depends_on` function, but are projected to the :doc:`test cases ` space. We will see the rules of that projection in a while. The dependency graph construction and the subsequent dependency analysis happen also at the level of the test cases. @@ -192,43 +192,6 @@ If you end up requiring such type of dependency in your tests, you might have to Technically, the framework could easily support such types of dependencies, but ReFrame's output would have to change substantially. - -Resolving dependencies ----------------------- - -As shown in the :doc:`tutorial_deps`, test dependencies would be of limited usage if you were not able to use the results or information of the target tests. -Let's reiterate over the :func:`set_executable` function of the :class:`OSULatencyTest` that we presented previously: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSULatencyTest.set_executable - -The ``@require_deps`` decorator does some magic -- we will unravel this shortly -- with the function arguments of the :func:`set_executable` function and binds them to the target test dependencies by their name. -However, as discussed in this section, dependencies are defined at test case level, so the ``OSUBuildTest`` function argument is bound to a special function that allows you to retrieve an actual test case of the target dependency. -This is why you need to "call" ``OSUBuildTest`` in order to retrieve the desired test case. -When no arguments are passed, this will retrieve the test case corresponding to the current partition and the current programming environment. -We could always retrieve the ``PrgEnv-gnu`` case by writing ``OSUBuildTest('PrgEnv-gnu')``. -If a dependency cannot be resolved, because it is invalid, a runtime error will be thrown with an appropriate message. - -The low-level method for retrieving a dependency is the :func:`getdep() ` method of the :class:`RegressionTest`. -In fact, you can rewrite :func:`set_executable` function as follows: - -.. code:: python - - @run_after('setup') - def set_executable(self): - target = self.getdep('OSUBuildTest') - self.executable = os.path.join( - target.stagedir, - 'osu-micro-benchmarks-5.6.2', 'mpi', 'pt2pt', 'osu_latency' - ) - self.executable_opts = ['-x', '100', '-i', '1000'] - - -Now it's easier to understand what the ``@require_deps`` decorator does behind the scenes. -It binds the function arguments to a partial realization of the :func:`getdep` function and attaches the decorated function as an after-setup hook. -In fact, any ``@require_deps``-decorated function will be invoked before any other after-setup hook. - - .. _cleaning-up-stage-files: Cleaning up stage files diff --git a/docs/howto.rst b/docs/howto.rst new file mode 100644 index 0000000000..cdc4c3d083 --- /dev/null +++ b/docs/howto.rst @@ -0,0 +1,1164 @@ +.. currentmodule:: reframe.core.pipeline.RegressionTest + +=============== +ReFrame How Tos +=============== + +This is a collection of "How To" articles on specific ReFrame usage topics. + + +.. contents:: Table of Contents + :local: + :depth: 3 + + +Working with build systems +========================== + +ReFrame supports building the test's code in many scenarios. +We have seen in the :doc:`tutorial` how to build the test's code if it is just a single file. +However, ReFrame knows how to interact with Make, CMake and Autotools. +Additionally, it supports integration with the `EasyBuild `__ build automation tool as well as the `Spack `__ package manager. +Finally, if none of the above build systems fits, users are allowed to use their custom build scripts. + + +Using Make, CMake or Autotools +------------------------------ + +We have seen already in the :ref:`tutorial ` how to build a test with a single source file. +ReFrame can also build test code using common build systems, such as `Make `__, `CMake `__ or `Autotools `__. +The build system to be used is selected by the :attr:`build_system` test attribute. +This is a "magic" attribute where you assign it a string and ReFrame will create the appropriate :ref:`build system object `. +Each build system can define its own properties, but some build systems have a common set of properties that are interpreted accordingly. +Let's see a version of the STREAM benchmark that uses ``make``: + +.. literalinclude:: ../examples/tutorial/stream/stream_make.py + :caption: + :lines: 5- + +Build system properties are set in a pre-compile hook. +In this case we set the CFLAGS and also pass Makefile target to the Make build system's :attr:`~reframe.core.buildsystems.Make.options`. + +.. warning:: + + You can't set build system options inside the test class body. + The test must be instantiated in order for the conversion from string to build system to happen. + The following will yield therefore an error: + + .. code-block:: python + + class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'Make' + build_system.flags = ['-O3'] + + +Based on the selected build system, ReFrame will generate the appropriate build script. + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_make.py -p gnu -r + cat output/tutorialsys/default/gnu/build_stream_40af02af/rfm_build.sh + +.. code-block:: bash + + #!/bin/bash + + _onerror() + { + exitcode=$? + echo "-reframe: command \`$BASH_COMMAND' failed (exit code: $exitcode)" + exit $exitcode + } + + trap _onerror ERR + + make -j 1 CC="gcc" CXX="g++" FC="ftn" NVCC="nvcc" CFLAGS="-O3 -fopenmp" stream_c.exe + + +Note that ReFrame passes several variables in the ``make`` command apart from those explicitly requested by the test, such as the ``CFLAGS``. +The rest of the flags are implicitly requested by the selected test environment, in this case ``gnu``, and ReFrame is trying its best to make sure that the environment's definition will be respected. +In the case of Autotools and CMake these variables will be set during the "configure" step. +Users can still override this behaviour and request explicitly to ignore any flags coming from the environment by setting the build system's :attr:`~reframe.core.buildsystems.BuildSystem.flags_from_environ` to :obj:`False`. +In this case, only the flags requested by the test will be emitted. + +The Autotools and CMake build systems are quite similar. +For passing ``configure`` options, the :attr:`~reframe.core.buildsystems.ConfigureBasedBuildSystem.config_opts` build system attribute should be set, whereas for ``make`` options the :attr:`~reframe.core.buildsystems.ConfigureBasedBuildSystem.make_opts` should be used. +The :ref:`OSU benchmarks ` in the main tutorial use the Autotools build system. + +Finally, in all three build systems, the :attr:`~reframe.core.buildsystems.Make.max_concurrency` can be set to control the number of parallel make jobs. + + +Integrating with EasyBuild +-------------------------- + +.. versionadded:: 3.5.0 + + +ReFrame integrates with the `EasyBuild `__ build automation framework, which allows you to use EasyBuild for building the source code of your test. + +Let's consider a simple ReFrame test that installs ``bzip2-1.0.6`` given the easyconfig `bzip2-1.0.6.eb `__ and checks that the installed version is correct. +The following code block shows the check, highlighting the lines specific to this tutorial: + +.. literalinclude:: ../examples/tutorial/easybuild/eb_test.py + :caption: + :start-at: import reframe + + +The test looks pretty standard except that we use the ``EasyBuild`` build system and set some build system-specific attributes. +More specifically, we set the :attr:`~reframe.core.buildsystems.EasyBuild.easyconfigs` attribute to the list of packages we want to build and install. +We also pass the ``-f`` option to EasyBuild's ``eb`` command through the :attr:`~reframe.core.buildsystems.EasyBuild.options` attribute, so that we force the build even if the corresponding environment module already exists. + +For running this test, we need the following Docker image: + +.. code-block:: bash + :caption: Run in the EasyBuild+Spack container. + + docker build -t reframe-eb-spack:latest -f examples/tutorial/dockerfiles/eb-spack.dockerfile . + docker run -h myhost --mount type=bind,source=$(pwd)/examples/,target=/home/user/reframe-examples --workdir=/home/user/reframe-examples/tutorial -it reframe-eb-spack:latest /bin/bash -l + + +EasyBuild requires a :ref:`modules system ` to run, so we need a configuration file that sets the modules system of the current system: + +.. literalinclude:: ../examples/tutorial/config/baseline_modules.py + :caption: + :lines: 5- + +We talk about modules system and how ReFrame interacts with them in :ref:`working-with-environment-modules`. +For the moment, we will only use them for running the EasyBuild example: + +.. code-block:: bash + :caption: Run in the EasyBuild+Spack container. + + reframe -C config/baseline_modules.py -c easybuild/eb_test.py -r + + +ReFrame generates the following commands to build and install the easyconfig: + +.. code-block:: bash + :caption: Run in the EasyBuild+Spack container. + + cat output/tutorialsys/default/builtin/BZip2EBCheck/rfm_build.sh + +.. code-block:: bash + + ... + export EASYBUILD_BUILDPATH=${stagedir}/easybuild/build + export EASYBUILD_INSTALLPATH=${stagedir}/easybuild + export EASYBUILD_PREFIX=${stagedir}/easybuild + export EASYBUILD_SOURCEPATH=${stagedir}/easybuild + eb bzip2-1.0.6.eb -f + +All the files generated by EasyBuild (sources, temporary files, installed software and the corresponding modules) are kept under the test's stage directory, thus the relevant EasyBuild environment variables are set. + +.. tip:: + + Users may set the EasyBuild prefix to a different location by setting the :attr:`~reframe.core.buildsystems.EasyBuild.prefix` attribute of the build system. + This allows you to have the built software installed upon successful completion of the build phase, but if the test fails in a later stage (sanity, performance), the installed software will not be cleaned up automatically. + +.. note:: + + ReFrame assumes that the ``eb`` executable is available on the system where the compilation is run (typically the local host where ReFrame is executed). + + +To run the freshly built package, the generated environment modules need to be loaded first. +These can be accessed through the :attr:`~reframe.core.buildsystems.EasyBuild.generated_modules` attribute *after* EasyBuild completes the installation. +For this reason, we set the test's :attr:`modules` in a pre-run hook. +This generated final run script is the following: + +.. code-block:: bash + :caption: Run in the EasyBuild+Spack container. + + cat output/tutorialsys/default/builtin/BZip2EBCheck/rfm_job.sh + +.. code-block:: bash + + module use ${stagedir}/easybuild/modules/all + module load bzip/1.0.6 + bzip2 --help + + +Packaging the installation +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The EasyBuild build system offers a way of packaging the installation via EasyBuild's packaging support. +To use this feature, `the FPM package manager `__ must be available. +By setting the dictionary :attr:`~reframe.core.buildsystems.Easybuild.package_opts` in the test, ReFrame will pass ``--package-{key}={val}`` to the EasyBuild invocation. +For instance, the following can be set to package the installations as an rpm file: + +.. code-block:: python + + self.keep_files = ['easybuild/packages'] + self.build_system.package_opts = { + 'type': 'rpm', + } + +The packages are generated by EasyBuild in the stage directory. +To retain them after the test succeeds, :attr:`~reframe.core.pipeline.RegressionTest.keep_files` needs to be set. + + +Integrating with Spack +---------------------- + +.. versionadded:: 3.6.1 + +ReFrame can also use `Spack `__ to build a software package and test it. + +The example shown here is the equivalent to the `EasyBuild <#integrating-with-easybuild>`__ one that built ``bzip2``. +Here is the test code: + +.. literalinclude:: ../examples/tutorial/spack/spack_test.py + :start-at: import reframe + + +When :attr:`~reframe.core.pipeline.RegressionTest.build_system` is set to ``'Spack'``, ReFrame will leverage Spack environments in order to build the test code. +By default, ReFrame will create a new Spack environment in the test's stage directory and add the requested :attr:`~reframe.core.buildsystems.Spack.specs` to it. + +.. note:: + Optional spec attributes, such as ``target`` and ``os``, should be specified in :attr:`~reframe.core.buildsystems.Spack.specs` and not as install options in :attr:`~reframe.core.buildsystems.Spack.install_opts`. + +You can set Spack configuration options for the new environment with the :attr:`~reframe.core.buildsystems.Spack.config_opts` attribute. These options take precedence over Spack's ``spack.yaml`` defaults. + +Users may also specify an existing Spack environment by setting the :attr:`~reframe.core.buildsystems.Spack.environment` attribute. +In this case, ReFrame treats the environment as a *test resource* so it expects to find it under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. + +To run this test, use the same container as with EasyBuild: + +.. code-block:: bash + :caption: Run in the EasyBuild+Spack container. + + docker run -h myhost --mount type=bind,source=$(pwd)/examples/,target=/home/user/reframe-examples --workdir=/home/user/reframe-examples/tutorial -it reframe-eb-spack:tutorial /bin/bash -l + +Conversely to EasyBuild, Spack does not require a modules systems to be configured, so you could simply run the test with ReFrame's builtin configuration: + +.. code-block:: bash + :caption: Run in the EasyBuild+Spack container. + + reframe -c spack/spack_test.py -r + +As with every other test, ReFrame will copy the test's resources to its stage directory before building it. +ReFrame will then activate the generated environment (either the one provided by the user or the one generated by ReFrame), add the given specs using the ``spack add`` command and, finally, install the packages in the environment. +Here is what ReFrame generates as a build script for this example: + +.. code:: bash + + spack env create -d rfm_spack_env + spack -e rfm_spack_env config add "config:install_tree:root:opt/spack" + spack -e rfm_spack_env add bzip2@1.0.6 + spack -e rfm_spack_env install + +As you might have noticed ReFrame expects that Spack is already installed on the system. +The packages specified in the environment and the tests will be installed in the test's stage directory, where the environment is copied before building. +Here is the stage directory structure: + +.. code:: console + + stage/generic/default/builtin/BZip2SpackCheck/ + ├── rfm_spack_env + │   ├── spack + │   │   └── opt + │   │      └── spack + │   │      ├── bin + │   │      └── darwin-catalina-skylake + │   ├── spack.lock + │   └── spack.yaml + ├── rfm_BZip2SpackCheck_build.err + ├── rfm_BZip2SpackCheck_build.out + ├── rfm_BZip2SpackCheck_build.sh + ├── rfm_BZip2SpackCheck_job.err + ├── rfm_BZip2SpackCheck_job.out + └── rfm_BZip2SpackCheck_job.sh + + +Finally, here is the generated run script that ReFrame uses to run the test, once its build has succeeded: + +.. code-block:: bash + + #!/bin/bash + spack env create -d rfm_spack_env + eval `spack -e rfm_spack_env load --sh bzip2@1.0.6` + bzip2 --help + +From this point on, sanity and performance checking are exactly identical to any other ReFrame test. + +.. tip:: + + While developing a test using Spack or EasyBuild as a build system, it can be useful to run ReFrame with the :option:`--keep-stage-files` and :option:`--dont-restage` options to prevent ReFrame from removing the test's stage directory upon successful completion of the test. + For this particular type of test, these options will avoid having to rebuild the required package dependencies every time the test is retried. + + + +Custom builds +------------- + +There are cases where you need to test a code that does not use of the supported build system of ReFrame. +In this case, you could set the :attr:`build_system` to ``'CustomBuild'`` and supply the exact commands to build the code: + + +.. code-block:: python + + @rfm.simple_test + class CustomBuildCheck(rfm.RegressionTest): + build_system = 'CustomBuild' + + @run_before('compile') + def setup_build(self): + self.build_system.commands = [ + './myconfigure.sh', + './build.sh' + ] + + +.. warning:: + + You should use this build system with caution, because environment management, reproducibility and any potential side effects are all controlled by the custom build system. + + +.. _working-with-environment-modules: + +Working with environment modules +================================ + +A common practice in HPC environments is to provide the software stack through `environment modules `__. +An environment module is essentially a set of environment variables that are sourced in the user's current shell in order to make available the requested software stack components. + +ReFrame allows users to associate an environment modules system to a system in the configuration file. +Tests may then specify the environment modules needed for them to run. + +We have seen environment modules in practice with the EasyBuild integration. +Systems that use environment modules must set the :attr:`~config.systems.modules_system` system configuration parameter to the modules system that the system uses. + +.. literalinclude:: ../examples/tutorial/config/baseline_modules.py + :lines: 5- + + +The tests that require environment modules must simply list the required modules in their :attr:`modules` variable. +ReFrame will then emit the correct commands to load the modules based on the configured modules system. +For older modules systems, such as Tmod 3.2, that do not support automatic conflict resolution, ReFrame will also emit commands to unload the conflicted modules before loading the requested ones. + +Test environments can also use modules by settings their :attr:`~config.environments.modules` parameter. + +.. code-block:: python + + 'environments': [ + ... + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'modules': ['gnu'], + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ... + ] + + +Environment module mappings +--------------------------- + +ReFrame allows you to replace environment modules used in tests with other modules on-the-fly. +This is quite useful if you want to test a new version of a module or another combination of modules. +Assume you have a test that loads a ``gromacs`` module: + +.. code-block:: python + + class GromacsTest(rfm.RunOnlyRegressionTest): + ... + modules = ['gromacs'] + + +This test would use the default version of the module in the system, but you might want to test another version, before making that new one the default. +You can ask ReFrame to temporarily replace the ``gromacs`` module with another one as follows: + + +.. code-block:: bash + + reframe -n GromacsTest -M 'gromacs:gromacs/2020.5' -r + + +Every time ReFrame tries to load the ``gromacs`` module, it will replace it with ``gromacs/2020.5``. +You can specify multiple mappings at once or provide a file with mappings using the :option:`--module-mappings` option. +You can also replace a single module with multiple modules. + +A very convenient feature of ReFrame in dealing with modules is that you do not have to care about module conflicts at all, regardless of the modules system backend. +ReFrame will take care of unloading any conflicting modules, if the underlying modules system cannot do that automatically. +In case of module mappings, it will also respect the module order of the replacement modules and will produce the correct series of "load" and "unload" commands needed by the modules system backend used. + + +Manipulating ReFrame's environment +---------------------------------- + +ReFrame runs the selected tests in the same environment as the one that it itself executes in. +It does not unload any environment modules nor sets or unsets any environment variable. +Nonetheless, it gives you the opportunity to modify the environment that the tests execute. +You can either purge completely all environment modules by passing the :option:`--purge-env` option or ask ReFrame to load or unload some environment modules before starting running any tests by using the :option:`-m` and :option:`-u` options respectively. +Of course you could manage the environment manually, but it's more convenient if you do that directly through ReFrame's command-line. +If you used an environment module to load ReFrame, e.g., ``reframe``, you can use the :option:`-u` to have ReFrame unload it before running any tests, so that the tests start in a clean environment: + +.. code-block:: bash + + reframe -u reframe ... + + +Working with low-level dependencies +=================================== + +We have seen that :ref:`test fixtures ` fixtures introduce dependencies between tests along with a scope. +It is possible to define test dependencies without a scope using the low-level test dependency API. +In fact, test fixtures translate to that low-level API. +In this how-to, we will rewrite the :ref:`OSU benchmarks example ` of the main tutorial to use the low-level dependency API. + +Here is the full code: + +.. literalinclude:: ../examples/tutorial/mpi/osu_deps.py + :caption: + :lines: 5- + +Contrary to when using fixtures, dependencies are now explicitly defined using the :func:`depends_on` method. +The target test is referenced by name and the option ``how`` argument defines how the individual cases of the two tests depend on each other. +Remember that a test generates a test case for each combination of valid systems and valid environments. +There are some shortcuts for defining common dependency patterns, such as the :obj:`udeps.fully` and :obj:`udeps.by_env`. +The former defines that all the test cases of the current test depend on all the test cases of the target, whereas the latter defines that test cases depend by environment, i.e., a test case of the current test depends on a test case of the target test only when the environment is the same. +In our example, the :obj:`build_osu_benchmarks` depends fully on the :obj:`fetch_osu_benchmarks` whereas the final benchmarks depend on the :obj:`build_os_benchmarks` by environment. +This is similar to the session and environment scopes of fixtures, but you have to set the :attr:`valid_systems` and :attr:`valid_prog_environs` of the targets, whereas for fixtures these will be automatically determined by the scope. +This makes the low-level dependencies less flexible. + +As with fixtures, you can still access fully the target test, but the way to do so is a bit more involved. +There are two ways to access the target dependencies: + +1. Using the :func:`@require_deps ` decorator. +2. Using the low-level :func:`getdep` method. + +The :func:`@require_deps ` acts as a special post-setup hook (in fact, it is always the first post-setup hook of the test) that binds each argument of the decorated function to the corresponding target dependency. +For the binding to work correctly, the function arguments must be named after the target dependencies. +However, referring to a dependency only by the test's name is not enough, since a test might be associated with multiple environments or partitions. +For this reason, each dependency argument is essentially bound to a function that accepts as argument the name of the target partition and target programming environment. +If no arguments are passed, the current programming environment is implied, such that ``build_osu_benchmarks()`` is equivalent to ``build_osu_benchmarks(self.current_environ.name, self.current_partition.name)``. +If the target partition and environment do not match the current ones, we should specify them, as is the case for accessing the :obj:`fetch_osu_benchmarks` dependency. +This call returns a handle to the actual target test object that, exactly as it happens when accessing the fixture handle in a post-setup hook. + +Target dependencies can also be accessed directly using the :func:`getdep` function. +This is what both the :func:`@require_deps ` decorator and fixtures use behind the scenes. +Let's rewrite the dependency hooks using the low-level :func:`getdep` function: + +.. code-block:: python + + @run_before('compile') + def prepare_build(self): + target = self.getdep('fetch_osu_benchmarks', 'gnu', 'login') + ... + + @run_before('run') + def prepare_run(self): + osu_binaries = self.getdep('build_osu_benchmarks') + ... + +For running and listing tests with dependencies the same principles apply as with fixtures as ReFrame only sees dependencies and test cases. +The only difference in listing is that there is no scope associated with the dependent tests as is with fixtures: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster_mpi.py -c mpi/osu_deps.py -n osu_allreduce_test -l + +.. code-block:: console + + [List of matched checks] + - osu_allreduce_test /63dd518c + ^build_osu_benchmarks /f6911c4c + ^fetch_osu_benchmarks /52d9b2c6 + Found 3 check(s) + + +Resolving dependencies +---------------------- + +When defining a low-level dependency using the :func:`depends_on` function, the target test cases must exist, otherwise ReFrame will refuse to load the dependency chain and will issue a warning. +Similarly, when requesting access to a target test case using :func:`getdep`, if the target test case does not exist, the current test will fail. + +To fully understand how the different cases of a test depend on the cases of another test and how to express more complex dependency relations, please refer to :doc:`dependencies`. +It is generally preferable to use the higher-level fixture API instead of the low-level dependencies as it's more intuitive, less error-prone and offers more flexibility. + + +.. _param_deps: + +Depending on parameterized tests +-------------------------------- + +As we have seen earlier, tests define their dependencies by referencing the target tests by their unique name. +This is straightforward when referring to regular tests, where their name matches the class name, but it becomes cumbersome trying to refer to a parameterized tests, since no safe assumption should be made as of the variant number of the test or how the parameters are encoded in the name. +In order to safely and reliably refer to a parameterized test, you should use the :func:`~reframe.core.pipeline.RegressionMixin.get_variant_nums` and :func:`~reframe.core.pipeline.RegressionMixin.variant_name` class methods as shown in the following example: + +.. literalinclude:: ../examples/tutorial/deps/parameterized.py + :lines: 6- + +In this example, :class:`TestB` depends only on selected variants of :class:`TestA`. +The :func:`get_variant_nums` method accepts a set of key-value pairs representing the target test parameters and selector functions and returns the list of the variant numbers that correspond to these variants. +Using the :func:`variant_name` subsequently, we can get the actual name of the variant. + + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -c deps/parameterized.py -l + +.. code-block:: console + + [List of matched checks] + - TestB /cc291487 + ^TestA %z=9 /ca1c96ee + ^TestA %z=8 /75b6718c + ^TestA %z=7 /1d87616c + ^TestA %z=6 /06c8e673 + - TestA %z=5 /536115e0 + - TestA %z=4 /b1aa0bc1 + - TestA %z=3 /e62d23e8 + - TestA %z=2 /423a76e9 + - TestA %z=1 /8258ae7a + - TestA %z=0 /7a14ae93 + Found 11 check(s) + + +.. _generate-ci-pipeline: + +Integrating into a CI pipeline +============================== + +.. versionadded:: 3.4.1 + +Instead of running your tests, you can ask ReFrame to generate a `child pipeline `__ specification for the Gitlab CI. +This will spawn a CI job for each ReFrame test respecting test dependencies. +You could run your tests in a single job of your Gitlab pipeline, but you would not take advantage of the parallelism across different CI jobs. +Having a separate CI job per test makes it also easier to spot the failing tests. + +As soon as you have set up a `runner `__ for your repository, it is fairly straightforward to use ReFrame to automatically generate the necessary CI steps. +The following is an example of ``.gitlab-ci.yml`` file that does exactly that: + +.. code-block:: yaml + + stages: + - generate + - test + + generate-pipeline: + stage: generate + script: + - reframe --ci-generate=${CI_PROJECT_DIR}/pipeline.yml -c ${CI_PROJECT_DIR}/path/to/tests + artifacts: + paths: + - ${CI_PROJECT_DIR}/pipeline.yml + + test-jobs: + stage: test + trigger: + include: + - artifact: pipeline.yml + job: generate-pipeline + strategy: depend + + +It defines two stages. +The first one, called ``generate``, will call ReFrame to generate the pipeline specification for the desired tests. +All the usual :ref:`test selection options ` can be used to select specific tests. +ReFrame will process them as usual, but instead of running the selected tests, it will generate the correct steps for running each test individually as a Gitlab job in a child pipeline. +The generated ReFrame command that will run each individual test reuses the :option:`-C`, :option:`-R`, :option:`-v` and :option:`--mode` options passed to the initial invocation of ReFrame that was used to generate the pipeline. +Users can define CI-specific execution modes in their configuration in order to pass arbitrary options to the ReFrame invocation in the child pipeline. + +Finally, we pass the generated CI pipeline file to the second phase as an artifact and we are done! +If ``image`` keyword is defined in ``.gitlab-ci.yml``, the emitted pipeline will use the same image as the one defined in the parent pipeline. +Besides, each job in the generated pipeline will output a separate junit report which can be used to create GitLab badges. + +The following figure shows one part of an automatically generated pipeline. + +.. figure:: _static/img/gitlab-ci.png + :align: center + + :sub:`Snapshot of a Gitlab pipeline generated automatically by ReFrame.` + + +.. note:: + + The ReFrame executable must be available in the Gitlab runner that will run the CI jobs. + + +Flexible tests +============== + +.. versionadded:: 2.15 + +ReFrame can automatically set the number of tasks of a particular test, if its :attr:`num_tasks` attribute is set to a negative value or zero. +In ReFrame's terminology, such tests are called *flexible*. +Negative values indicate the minimum number of tasks that are acceptable for this test (a value of ``-4`` indicates that at least ``4`` tasks are required). +A zero value indicates the default minimum number of tasks which is equal to :attr:`num_tasks_per_node`. + +By default, ReFrame will spawn such a test on all the idle nodes of the current system partition, but this behavior can be adjusted with :option:`--flex-alloc-nodes` command-line option. +Flexible tests are very useful for multi-node diagnostic tests. + +In this example, we demonstrate this feature by forcing flexible execution in the OSU allreduce benchmark. + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster_mpi.py -c mpi/osu.py -n osu_allreduce_test -S num_tasks=0 -r + +By default, our version of the OSU allreduce benchmark uses two processes, but setting :attr:`num_tasks` to zero will span the test to the full pseudo-cluster occupying all three available nodes: + +.. code-block:: console + + admin@login:~$ squeue + JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 5 all rfm_osu_ admin R 1:04 3 nid[00-02] + +Note that for flexible tests, :attr:`num_tasks` is updated to the actual value of tasks that ReFrame requested just after the test job is submitted. +Thus, the actual number of tasks can then be used in sanity or performance checking. + +.. tip:: + + If you want to run multiple flexible tests at once that compete for the same nodes, you will have to run them using the serial execution policy, because the first test will take all the available idel nodes causing the rest to fail immediately, as there will be no available nodes for them. + + +Testing containerized applications +================================== + +.. versionadded:: 2.20 + +ReFrame can be used also to test applications that run inside a container. +First, you will need to enable the container platform support in ReFrame's configuration: + +.. literalinclude:: ../examples/tutorial/config/baseline_contplatf.py + :caption: + :lines: 5- + +For each partition, users can define a list of all supported container platforms using the :attr:`~config.systems.partitions.container_platforms` configuration parameter. +In this case define the `Docker `__ platform. +If your system supports multiple configuration platforms, ReFrame offers more configuration options, such as setting up the environment or indicating which platform is the default one. + +To denote that a test should be launched inside a container, the test must set the :attr:`container_platform` variable. +Here is an example: + + +.. literalinclude:: ../examples/tutorial/containers/container_test.py + :caption: + :lines: 5- + +A container-based test should be written as a :class:`RunOnlyRegressionTest`. +The :attr:`container_platform` variable accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. +It is not necessary to set this variable, in which case, the default container platform of the current partition will be used. +You can still differentiate your test based on the actual container platform that is being used by checking the ``self.container_platform.name`` variable. + +As soon as the container platform to be used is determined, you need to specify the container image to use by setting the :attr:`~reframe.core.containers.ContainerPlatform.image`. +If the image is not specified, then the container logic is skipped and the test executes as if the :attr:`container_platform` was never set. + +The :attr:`~reframe.core.containers.ContainerPlatform.image` is the only mandatory attribute for container-based checks. +It is important to note that the :attr:`executable` and :attr:`executable_opts` attributes of the actual test are ignored if the containerized code path is taken, i.e., when :attr:`~reframe.core.containers.ContainerPlatform.image` is not :obj:`None`. + +Running the test, ReFrame will generate a script that will launch and run the container for the given platform: + +.. note:: + + This example must be run natively. + + +.. code-block:: bash + :caption: Run locally. + + reframe -C examples/tutorial/config/baseline_contplatf.py -c examples/tutorial/containers/container_test.py -r + +And this is the generated test job script: + +.. code-block:: bash + + #!/bin/bash + docker pull ubuntu:18.04 + docker run --rm -v "/Users/karakasv/Repositories/reframe/stage/tutorialsys/default/builtin/ContainerTest":"/rfm_workdir" -w /rfm_workdir ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' + +By default, ReFrame will pull the image, but this can be skipped by setting the :attr:`container_platform` 's :attr:`~reframe.core.containers.ContainerPlatform.pull_image` attribute to :obj:`False`. +Also, ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container. +Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. +Besides the stage directory, additional mount points can be specified through the :attr:`~reframe.core.pipeline.RegressionTest.container_platform.mount_points` :attr:`container_platform` attribute: + +.. code-block:: python + + self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), + ('/path/to/host/dir2', '/path/to/container/mount_point2')] + +The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy artifacts as needed. +These artifacts will therefore be available inside the stage directory after the container execution finishes. +This is very useful if the artifacts are needed for the sanity or performance checks. +If the copy is not performed by the default container command, the user can override this command by settings the :attr:`container_platform` 's :attr:`~reframe.core.containers.ContainerPlatform.command` such as to include the appropriate copy commands. +In the current test, the output of the ``cat /etc/os-release`` is available both in the standard output as well as in the ``release.txt`` file, since we have used the command: + +.. code-block:: bash + + bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' + + +and ``/rfm_workdir`` corresponds to the stage directory on the host system. +Therefore, the ``release.txt`` file can now be used in the subsequent sanity checks: + +.. literalinclude:: ../examples/tutorial/containers/container_test.py + :start-at: @sanity_function + :end-at: return + + + +.. versionchanged:: 3.12.0 + There is no need any more to explicitly set the :attr:`container_platform` in the test. + This is automatically initialized from the default platform of the current partition. + + +Generating tests programmatically +================================= + +You can use ReFrame to generate tests programmatically using the special :func:`~reframe.core.meta.make_test` function. +This function creates a new test type as if you have typed it manually using the :keyword:`class` keyword. +You can create arbitrarily complex tests that use variables, parameters, fixtures and pipeline hooks. + +In this tutorial, we will use :func:`~reframe.core.meta.make_test` to build a simple domain-specific syntax for generating variants of STREAM benchmarks. +Our baseline STREAM test is the one presented in the :doc:`tutorial` that uses a build fixture: + +.. literalinclude:: ../examples/tutorial/stream/stream_fixtures.py + :caption: + :lines: 5- + +For our example, we would like to create a simpler syntax for generating multiple different :class:`stream_test` versions that could run all at once. +Here is an example specification file for those tests: + +.. literalinclude:: ../examples/tutorial/stream/stream_config.yaml + :caption: + :lines: 6- + + +The :attr:`thread_scaling` configuration parameter for the last workflow will create a parameterised version of the test using different number of threads. +In total, we expect six :class:`stream_test` versions to be generated by this configuration. + +The process for generating the actual tests from this spec file comprises three steps and everything happens in a somewhat unconventional, though valid, ReFrame test file: + +1. We load the test configuration from a spec file that is passed through the ``STREAM_SPEC_FILE`` environment variable. +2. Based on the loaded test specs, we generate the actual tests using the :func:`~reframe.core.meta.make_test` function. +3. We register the generated tests with the framework by applying manually the :func:`@simple_test ` decorator. + +The whole code for generating the tests is the following and is only a few lines. +Let's walk through it. + +.. literalinclude:: ../examples/tutorial/stream/stream_workflows.py + :caption: + :lines: 5- + +The :func:`load_specs()` function simply loads the test specs from the YAML test spec file and does some simple sanity checking. + +The :func:`generate_tests()` function consumes the test specs and generates a test for each entry. +Each test inherits from the base :class:`stream_test` and redefines its :attr:`stream_binaries` fixture so that it is instantiated with the set of variables specified in the test spec. +Remember that all the STREAM test variables in the YAML file refer to its build phase and thus its build fixture. +We also treat specially the :attr:`thread_scaling` spec parameter. +In this case, we add a :attr:`num_threads` parameter to the test and add a post-init hook that sets the test's :attr:`~reframe.core.pipeline.RegressionTest.num_cpus_per_task`. + +Finally, we register the generated tests using the :func:`rfm.simple_test` decorator directly; +remember that :func:`~reframe.core.meta.make_test` returns a class. + +The equivalent of our test generation for the third spec is exactly the following: + +.. code-block:: python + + @rfm.simple_test + class stream_test_2(stream_test): + stream_binary = fixture(build_stream, scope='environment', + variables={'elem_type': 'double', + 'array_size': 16777216, + 'num_iters': 10}) + nthr = parameter([1, 2, 4, 8]) + + @run_after('init') + def _set_num_threads(self): + self.num_threads = self.nthr + + +And here is the listing of generated tests: + +.. code-block:: bash + :caption: Run in the single-node container. + + STREAM_SPEC_FILE=stream_config.yaml reframe -C config/baseline_environs.py -c stream/stream_workflows.py -l + +.. code-block:: console + + [List of matched checks] + - stream_test_2 %nthr=8 %stream_binary.elem_type=double %stream_binary.array_size=16777216 %stream_binary.num_iters=10 /04f5cf62 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+gnu 'stream_binary /74d12df7 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+clang 'stream_binary /f3a963e3 + - stream_test_2 %nthr=4 %stream_binary.elem_type=double %stream_binary.array_size=16777216 %stream_binary.num_iters=10 /1c09d755 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+gnu 'stream_binary /74d12df7 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+clang 'stream_binary /f3a963e3 + - stream_test_2 %nthr=2 %stream_binary.elem_type=double %stream_binary.array_size=16777216 %stream_binary.num_iters=10 /acb6dc4d + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+gnu 'stream_binary /74d12df7 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+clang 'stream_binary /f3a963e3 + - stream_test_2 %nthr=1 %stream_binary.elem_type=double %stream_binary.array_size=16777216 %stream_binary.num_iters=10 /e6eebc18 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+gnu 'stream_binary /74d12df7 + ^build_stream %elem_type=double %array_size=16777216 %num_iters=10 ~tutorialsys:default+clang 'stream_binary /f3a963e3 + - stream_test_1 %stream_binary.elem_type=double %stream_binary.array_size=1048576 %stream_binary.num_iters=100 /514be749 + ^build_stream %elem_type=double %array_size=1048576 %num_iters=100 ~tutorialsys:default+gnu 'stream_binary /b841f3c9 + ^build_stream %elem_type=double %array_size=1048576 %num_iters=100 ~tutorialsys:default+clang 'stream_binary /ade049de + - stream_test_0 %stream_binary.elem_type=float %stream_binary.array_size=16777216 %stream_binary.num_iters=10 /c0c0f2bf + ^build_stream %elem_type=float %array_size=16777216 %num_iters=10 ~tutorialsys:default+gnu 'stream_binary /6767ce8c + ^build_stream %elem_type=float %array_size=16777216 %num_iters=10 ~tutorialsys:default+clang 'stream_binary /246007ff + Found 6 check(s) + + +.. note:: + + The path passed to ``STREAM_SPEC_FILE`` is relative to the test directory. + Since version 4.2, ReFrame changes to the test directory before loading a test file. + In prior versions you have to specify the path relative to the current working directory. + + +Using the Flux framework scheduler +================================== + +This is a how to that will show how-to use ReFrame with `Flux +Framework `__. First, build the +container here from the root of reframe. + +.. code-block:: bash + + $ docker build -f examples/tutorial/dockerfiles/flux.dockerfile -t flux-reframe . + +Then shell inside, optionally binding the present working directory if +you want to develop. + +.. code-block:: bash + + $ docker run -it -v $PWD:/code flux-reframe + $ docker run -it flux-reframe + +Note that if you build the local repository, you’ll need to bootstrap +and install again, as we have over-written the bin! + +.. code-block:: bash + + # In case of problems with pip, first clean the `external` directory with `rm -rf external` + ./bootstrap.sh + +And then reframe will again be in the local ``bin`` directory: + +.. code-block:: bash + + # which reframe + /code/bin/reframe + +Then we can run ReFrame with the custom config `config.py `__ +for flux. + +.. code-block:: bash + + # What tests are under examples/howto/flux? + $ cd examples/howto/flux + $ reframe -c . -C settings.py -l + +.. code-block:: console + + [ReFrame Setup] + version: 4.0.0-dev.1 + command: '/code/bin/reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l' + launched by: root@b1f6650222bc + working directory: '/code' + settings file: 'examples/howto/flux/settings.py' + check search path: '/code/examples/howto/flux' + stage directory: '/code/stage' + output directory: '/code/output' + + [List of matched checks] + - EchoRandTest /66b93401 + Found 1 check(s) + + Log file(s) saved in '/tmp/rfm-ilqg7fqg.log' + +This also works + +.. code-block:: bash + :caption: Run in the Flux container. + + $ reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l + +And then to run tests, just replace ``-l`` (for list) with ``-r`` or +``--run`` (for run): + +.. code-block:: bash + :caption: Run in the Flux container. + + $ reframe -c examples/howto/flux -C examples/howto/flux/settings.py --run + +.. code:: console + + root@b1f6650222bc:/code# reframe -c examples/howto/flux -C examples/howto/flux/settings.py --run + [ReFrame Setup] + version: 4.0.0-dev.1 + command: '/code/bin/reframe -c examples/howto/flux -C examples/howto/flux/settings.py --run' + launched by: root@b1f6650222bc + working directory: '/code' + settings file: 'examples/howto/flux/settings.py' + check search path: '/code/examples/howto/flux' + stage directory: '/code/stage' + output directory: '/code/output' + + [==========] Running 1 check(s) + [==========] Started on Fri Sep 16 20:47:15 2022 + + [----------] start processing checks + [ RUN ] EchoRandTest /66b93401 @generic:default+builtin + [ OK ] (1/1) EchoRandTest /66b93401 @generic:default+builtin + [----------] all spawned checks have finished + + [ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped) + [==========] Finished on Fri Sep 16 20:47:15 2022 + Run report saved in '/root/.reframe/reports/run-report.json' + Log file(s) saved in '/tmp/rfm-0avso9nb.log' + +For advanced users or developers, here is how to run tests within the container: + +Testing +------- + +.. code-block:: console + + ./test_reframe.py --rfm-user-config=examples/howto/flux/settings.py unittests/test_schedulers.py -xs + + +Building test libraries and utilities +===================================== + +ReFrame tests are extremely modular. +You can create libraries of base tests and utilities that others can use or extend. +You can organize the source code of a test library as you would with a regular Python code. +Let's see a made-up example for demonstration purposes: + +.. code-block:: console + + ~/reframe-examples/howto + ├── testlib + │   ├── __init__.py + │   ├── simple.py + │   └── utility + │   └── __init__.py + └── testlib_example.py + + +The ``testlib_example.py`` is fairly simple: +it extends the :class:`simple_echo_check` from the test library and sets the message. + +.. literalinclude:: ../examples/howto/testlib_example.py + :caption: + :lines: 6- + +The :class:`simple_echo_check` it echoes "Hello, " and asserts the output. +It also uses a dummy fixture that it includes from a utility. + +.. literalinclude:: ../examples/howto/testlib/simple.py + :caption: + :lines: 6- + +Note that the :class:`simple_echo_check` is also decorated as a :func:`@simple_test `, meaning that it can be executed as a stand-alone check. +This is typical when you are building test libraries: +you want the base tests to be complete and functional making minimum assumptions for the target system/environment. +You can then specialize further the derived tests and add more constraints in their :attr:`valid_systems` or :attr:`valid_prog_environs`. + +Let's try running both the library and the derived tests: + +.. code-block:: bash + :caption: Running the derived test (in the single-node container) + + reframe -c reframe-examples/howto/testlib_example.py -r + +.. code-block:: console + + [----------] start processing checks + [ RUN ] dummy_fixture ~generic:default+builtin /1fae4a8b @generic:default+builtin + [ OK ] (1/2) dummy_fixture ~generic:default+builtin /1fae4a8b @generic:default+builtin + [ RUN ] HelloFoo /2ecd9f04 @generic:default+builtin + [ OK ] (2/2) HelloFoo /2ecd9f04 @generic:default+builtin + [----------] all spawned checks have finished + +.. code-block:: bash + :caption: Running the library test (in the single-node container) + + reframe -c reframe-examples/howto/testlib/simple.py -r + +.. code-block:: console + + [----------] start processing checks + [ RUN ] dummy_fixture ~generic:default+builtin /1fae4a8b @generic:default+builtin + [ OK ] (1/2) dummy_fixture ~generic:default+builtin /1fae4a8b @generic:default+builtin + [ RUN ] simple_echo_check /8e1b0090 @generic:default+builtin + [ OK ] (2/2) simple_echo_check /8e1b0090 @generic:default+builtin + [----------] all spawned checks have finished + + +There is a little trick that makes running both the library test and the derived test so painlessly, despite the relative import of the :obj:`utility` module by the library test. +ReFrame loads the test files by importing them as Python modules using the file's basename as the module name. +It also adds temporarily to the ``sys.path`` the parent directory of the test file. +This is enough to load the ``testlib.simple`` module in the ``testlib_example.py`` and since the ``simple`` module has a parent, Python knows how to resolve the relative import in ``from .utility import dummy_fixture`` (it will be resolved as ``testlib.utility``). +However, loading directly the test library file, Python would not know the parent module of ``utility`` and would complain. +The trick is to create an empty ``testlib/__init__.py`` file, so as to tell ReFrame to load also ``testlib`` as a parent module. +Whenever ReFrame encounters an ``__init__.py`` file down the directory path leading to a test file, it will load it as a parent module, thus allowing relative imports to succeed. + + +Debugging +========= + +ReFrame tests are Python classes inside Python source files, so the usual debugging techniques for Python apply. +However, ReFrame will filter some errors and stack traces by default in order to keep the output clean. +Generally, full stack traces for user programming errors will not be printed and will not block the test loading process. +If a test has errors and cannot be loaded, an error message will be printed and the loading of the remaining tests will continue. +In the following, we have inserted a small typo in the ``stream_variables.py`` tutorial example: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables.py -l + +.. code-block:: console + + WARNING: skipping test file '/home/user/reframe-examples/tutorial/stream/stream_variables.py': name error: stream/stream_variables.py:30: name 'varible' is not defined + num_threads = varible(int, value=0) + (rerun with '-v' for more information) + +Rerunning with increased verbosity as the message suggests will give a full traceback. + +.. note:: + + ReFrame cannot always track a user error back to its source, especially for some of the `builtin `__ functionality. + In such cases, ReFrame will just print the error message but not the source code context. + +.. tip:: + The :option:`-v` option can be specified multiple times to increase the verbosity level further. + + +Debugging sanity and performance patterns +----------------------------------------- + +When creating a new test that requires a complex output parsing for the sanity checking or for extracting the figures of merit, tuning the functions decorated by :attr:`@sanity_function` or :attr:`@performance_function` may involve some trial and error to debug the complex regular expressions required. +For lightweight tests which execute in few seconds, this trial and error may not be an issue at all. +However, when dealing with tests which take longer to run, this method can quickly become tedious and inefficient. + +.. tip:: + When dealing with ``make``-based projects which take a long time to compile, you can use the command line option :option:`--dont-restage` in order to speed up the compile stage in subsequent runs. + +When a test fails, ReFrame will keep the test output in the stage directory after its execution, which means that one can load this output into a Python shell or another helper script without having to rerun the expensive test again. +If the test is not failing but the user still wants to experiment or modify the existing sanity or performance functions, the command line option :option:`--keep-stage-files` can be used when running ReFrame to avoid deleting the stage directory. +With the executable's output available in the stage directory, one can simply use the `re `_ module to debug regular expressions as shown below. + +.. code-block:: python + + >>> import re + + >>> # Read the test's output + >>> with open(the_output_file, 'r') as f: + ... test_output = ''.join(f.readlines()) + ... + >>> # Evaluate the regular expression + >>> re.findall(the_regex_pattern, test_output, re.MULTILINE) + +Alternatively to using the `re `_ module, one could use all the :mod:`~reframe.utility.sanity` utility provided by ReFrame directly from the Python shell. +In order to do so, if ReFrame was installed manually using the ``bootstrap.sh`` script, one will have to make all the Python modules from the ``external`` directory accessible to the Python shell as shown below. + +.. code-block:: python + + >>> import sys + >>> import os + + >>> # Make ReFrame's dependencies available + >>> sys.path = ['/path/to/reframe/prefix/external'] + sys.path + + >>> # Import ReFrame-provided sanity functions + >>> import reframe.utility.sanity as sn + + >>> # Evaluate the regular expression + >>> assert sn.evaluate(sn.assert_found(the_regex_pattern, the_output_file)) + + +Debugging test loading +---------------------- + +If you are new to ReFrame, you might wonder sometimes why your tests are not loading or why your tests are not running on the partition they were supposed to run. +This can be due to ReFrame picking the wrong configuration entry or that your test is not written properly (not decorated, no :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` etc.). +If you try to load a test file and list its tests by increasing twice the verbosity level, you will get enough output to help you debug such issues. +Let's try loading the ``stream_variables.py`` file: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables.py -l -vv + +.. literalinclude:: listings/verbose_test_loading.txt + :language: console + +You can see all the different phases ReFrame's frontend goes through when loading a test. +After loading the configuration, ReFrame will print out its relevant environment variables and will start examining the given files in order to find and load ReFrame tests. +Before attempting to load a file, it will validate it and check if it looks like a ReFrame test. +If it does, it will load that file by importing it. +This is where any ReFrame tests are instantiated and initialized (see ``Loaded 3 test(s)``), as well as the actual test cases (combination of tests, system partitions and environments) are generated. +Then the test cases are filtered based on the various :ref:`filtering command line options ` as well as the programming environments that are defined for the currently selected system. +Finally, the test case dependency graph is built and everything is ready for running (or listing). + +Try passing a specific system or partition with the :option:`--system` option or modify the test (e.g., removing the decorator that registers it) and see how the logs change. + + + +Extending the framework +======================= + +.. _custom-launchers: + +Implementing a parallel launcher backend +---------------------------------------- + +It is not uncommon for sites to supply their own alternatives of parallel launchers that build on top of existing launchers and provide additional functionality or implement some specific site policies. +In ReFrame it is straightforward to implement a custom parallel launcher backend without having to modify the framework code. + +Let's see how a builtin launcher looks like. +The following is the actual implementation of the ``mpirun`` launcher in ReFrame: + +.. literalinclude:: ../reframe/core/launchers/mpi.py + :pyobject: MpirunLauncher + + +Each launcher must derive from the abstract base class :class:`~reframe.core.launchers.JobLauncher` ands needs to implement the :func:`~reframe.core.launchers.JobLauncher.command` method and, optionally, change the default :func:`~reframe.core.launchers.JobLauncher.run_command` method. + +The :func:`~reframe.core.launchers.JobLauncher.command` returns a list of command tokens that will be combined with any user-supplied job launcher :attr:`~reframe.core.launchers.JobLauncher.options` by the :func:`~reframe.core.launchers.JobLauncher.run_command` method to generate the actual launcher command line. +Notice you can use the ``job`` argument to get job-specific information that will allow you to construct the correct launcher invocation. + +If you use a Python-based configuration file, you can define your custom launcher directly inside your config as follows: + +.. code-block:: python + + from reframe.core.backends import register_launcher + from reframe.core.launchers import JobLauncher + + + @register_launcher('slrun') + class MySmartLauncher(JobLauncher): + def command(self, job): + return ['slrun', ...] + + site_configuration = { + 'systems': [ + { + 'name': 'my_system', + 'partitions': [ + { + 'name': 'my_partition', + 'launcher': 'slrun' + ... + } + ], + ... + }, + ... + ], + ... + } + + +.. note:: + + In versions prior to 4.0, launchers could only be implemented inside the source code tree of ReFrame. diff --git a/docs/index.rst b/docs/index.rst index de85eca7f5..6d35d027a8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ Finally, ReFrame offers a powerful and efficient runtime for running and managin Publications ============ +* Slides [`pdf `__][`talk `__] @ `9th EasyBuild User Meeting 2024 `__. * Slides [`part 1 `__][`part 2 `__][`talk `__] @ `8th EasyBuild User Meeting 2023 `__. * Slides [`pdf `__] @ `7th EasyBuild User Meeting 2022 `__. * Slides [`pdf `__] @ `6th EasyBuild User Meeting 2021 `__. @@ -44,12 +45,11 @@ Webinars and Tutorials .. toctree:: :caption: Table of Contents - :maxdepth: 2 started - whats_new_40 - tutorials - configure + tutorial + howto topics manuals + whats_new_40 hpctestlib diff --git a/docs/listings/alltests_daint.txt b/docs/listings/alltests_daint.txt deleted file mode 100644 index 5f2f96c200..0000000000 --- a/docs/listings/alltests_daint.txt +++ /dev/null @@ -1,146 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/basics/ -R -n HelloMultiLangTest|HelloThreadedExtended2Test|StreamWithRefTest --performance-report -r' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: (R) '/home/user/Devel/reframe/tutorials/basics' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-nyqs7jb9.log' - -[==========] Running 4 check(s) -[==========] Started on Tue Nov 15 18:20:32 2022 - -[----------] start processing checks -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+builtin -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+gnu -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+intel -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+nvidia -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+cray -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+gnu -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+intel -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+nvidia -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+cray -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+gnu -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+intel -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+nvidia -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+cray -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:login+builtin -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:login+gnu -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:login+intel -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:login+nvidia -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:login+cray -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+gnu -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+intel -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+nvidia -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+cray -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:mc+gnu -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:mc+intel -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:mc+nvidia -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @daint:mc+cray -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:login+builtin -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:login+gnu -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:login+intel -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:login+nvidia -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:login+cray -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:gpu+gnu -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:gpu+intel -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:gpu+nvidia -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:gpu+cray -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:mc+gnu -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:mc+intel -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:mc+nvidia -[ RUN ] HelloThreadedExtended2Test /57223829 @daint:mc+cray -[ RUN ] StreamWithRefTest /f925207b @daint:login+gnu -[ RUN ] StreamWithRefTest /f925207b @daint:gpu+gnu -[ RUN ] StreamWithRefTest /f925207b @daint:mc+gnu -[ OK ] ( 1/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+builtin -[ OK ] ( 2/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+gnu -[ OK ] ( 3/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+intel -[ OK ] ( 4/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+nvidia -[ OK ] ( 5/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:login+cray -[ OK ] ( 6/42) HelloMultiLangTest %lang=c /7cfa870e @daint:login+builtin -[ OK ] ( 7/42) HelloMultiLangTest %lang=c /7cfa870e @daint:login+gnu -[ OK ] ( 8/42) HelloMultiLangTest %lang=c /7cfa870e @daint:login+intel -[ OK ] ( 9/42) HelloMultiLangTest %lang=c /7cfa870e @daint:login+nvidia -[ OK ] (10/42) HelloMultiLangTest %lang=c /7cfa870e @daint:login+cray -[ OK ] (11/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+cray -[ OK ] (12/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+nvidia -[ OK ] (13/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+cray -[ OK ] (14/42) HelloMultiLangTest %lang=c /7cfa870e @daint:mc+cray -[ OK ] (15/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+nvidia -[ OK ] (16/42) HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+intel -[ OK ] (17/42) HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+nvidia -[ OK ] (18/42) HelloMultiLangTest %lang=c /7cfa870e @daint:mc+intel -[ OK ] (19/42) HelloThreadedExtended2Test /57223829 @daint:login+builtin -[ OK ] (20/42) HelloThreadedExtended2Test /57223829 @daint:login+gnu -[ OK ] (21/42) HelloThreadedExtended2Test /57223829 @daint:login+intel -[ OK ] (22/42) HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+cray -[ OK ] (23/42) HelloMultiLangTest %lang=c /7cfa870e @daint:mc+gnu -[ OK ] (24/42) HelloThreadedExtended2Test /57223829 @daint:login+nvidia -[ OK ] (25/42) HelloThreadedExtended2Test /57223829 @daint:login+cray -[ OK ] (26/42) HelloMultiLangTest %lang=c /7cfa870e @daint:mc+nvidia -[ OK ] (27/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+gnu -[ OK ] (28/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:gpu+intel -[ OK ] (29/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+gnu -[ OK ] (30/42) HelloMultiLangTest %lang=cpp /71bf65a3 @daint:mc+intel -[ OK ] (31/42) HelloMultiLangTest %lang=c /7cfa870e @daint:gpu+gnu -[ OK ] (32/42) StreamWithRefTest /f925207b @daint:login+gnu -P: Copy: 71061.6 MB/s (r:0, l:None, u:None) -P: Scale: 44201.5 MB/s (r:0, l:None, u:None) -P: Add: 48178.5 MB/s (r:0, l:None, u:None) -P: Triad: 48063.3 MB/s (r:0, l:None, u:None) -[ OK ] (33/42) HelloThreadedExtended2Test /57223829 @daint:mc+cray -[ OK ] (34/42) HelloThreadedExtended2Test /57223829 @daint:mc+intel -[ OK ] (35/42) HelloThreadedExtended2Test /57223829 @daint:mc+gnu -[ OK ] (36/42) HelloThreadedExtended2Test /57223829 @daint:mc+nvidia -[ OK ] (37/42) StreamWithRefTest /f925207b @daint:mc+gnu -P: Copy: 52660.1 MB/s (r:0, l:None, u:None) -P: Scale: 33117.6 MB/s (r:0, l:None, u:None) -P: Add: 34876.7 MB/s (r:0, l:None, u:None) -P: Triad: 35150.7 MB/s (r:0, l:None, u:None) -[ OK ] (38/42) HelloThreadedExtended2Test /57223829 @daint:gpu+intel -[ OK ] (39/42) HelloThreadedExtended2Test /57223829 @daint:gpu+cray -[ OK ] (40/42) HelloThreadedExtended2Test /57223829 @daint:gpu+nvidia -[ OK ] (41/42) HelloThreadedExtended2Test /57223829 @daint:gpu+gnu -[ OK ] (42/42) StreamWithRefTest /f925207b @daint:gpu+gnu -P: Copy: 49682.3 MB/s (r:0, l:None, u:None) -P: Scale: 34452.3 MB/s (r:0, l:None, u:None) -P: Add: 38030.7 MB/s (r:0, l:None, u:None) -P: Triad: 38379.0 MB/s (r:0, l:None, u:None) -[----------] all spawned checks have finished - -[ PASSED ] Ran 42/42 test case(s) from 4 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Tue Nov 15 18:22:48 2022 - -================================================================================ -PERFORMANCE REPORT --------------------------------------------------------------------------------- -[StreamWithRefTest /f925207b @daint:login:gnu] - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 71061.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 44201.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 48178.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 48063.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamWithRefTest /f925207b @daint:gpu:gnu] - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 49682.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 34452.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 38030.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 38379.0 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamWithRefTest /f925207b @daint:mc:gnu] - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 52660.1 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 33117.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 34876.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 35150.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) --------------------------------------------------------------------------------- -Run report saved in '/home/user/.reframe/reports/run-report-1.json' -Log file(s) saved in '/tmp/rfm-nyqs7jb9.log' diff --git a/docs/listings/deps_complex_run.txt b/docs/listings/deps_complex_run.txt index a8fe0ac662..b575f42ea9 100644 --- a/docs/listings/deps_complex_run.txt +++ b/docs/listings/deps_complex_run.txt @@ -1,211 +1,116 @@ [ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-_008n_el.log' + version: 4.6.0-dev.2 + command: '/usr/local/share/reframe/bin/reframe -c deps/deps_complex.py -r --nocolor' + launched by: user@myhost + working directory: '/home/user/reframe-examples/tutorial' + settings files: '' + check search path: '/home/user/reframe-examples/tutorial/deps/deps_complex.py' + stage directory: '/home/user/reframe-examples/tutorial/stage' + output directory: '/home/user/reframe-examples/tutorial/output' + log files: '/tmp/rfm-01gkxmq0.log' [==========] Running 10 check(s) -[==========] Started on Sat Nov 12 19:01:00 2022 +[==========] Started on Tue Apr 16 21:35:34 2024+0000 [----------] start processing checks -[ RUN ] T0 /c9c2be9f @tresa:default+gnu -[ RUN ] T0 /c9c2be9f @tresa:default+clang -[ OK ] ( 1/20) T0 /c9c2be9f @tresa:default+gnu -[ OK ] ( 2/20) T0 /c9c2be9f @tresa:default+clang -[ RUN ] T4 /11ee5e9a @tresa:default+gnu -[ RUN ] T4 /11ee5e9a @tresa:default+clang -[ OK ] ( 3/20) T4 /11ee5e9a @tresa:default+gnu -[ OK ] ( 4/20) T4 /11ee5e9a @tresa:default+clang -[ RUN ] T5 /020d01e5 @tresa:default+gnu -[ RUN ] T5 /020d01e5 @tresa:default+clang -[ OK ] ( 5/20) T5 /020d01e5 @tresa:default+gnu -[ OK ] ( 6/20) T5 /020d01e5 @tresa:default+clang -[ RUN ] T1 /1f93603d @tresa:default+gnu -[ RUN ] T1 /1f93603d @tresa:default+clang -[ OK ] ( 7/20) T1 /1f93603d @tresa:default+gnu -[ OK ] ( 8/20) T1 /1f93603d @tresa:default+clang -[ RUN ] T8 /605fc1d6 @tresa:default+gnu -[ FAIL ] ( 9/20) T8 /605fc1d6 @tresa:default+gnu -==> test failed during 'setup': test staged in '/home/user/Repositories/reframe/stage/tresa/default/gnu/T8' -[ RUN ] T8 /605fc1d6 @tresa:default+clang -[ FAIL ] (10/20) T8 /605fc1d6 @tresa:default+clang -==> test failed during 'setup': test staged in '/home/user/Repositories/reframe/stage/tresa/default/clang/T8' -[ FAIL ] (11/20) T9 /78a78a4e @tresa:default+gnu +[ RUN ] T0 /c9c2be9f @generic:default+builtin +[ OK ] ( 1/10) T0 /c9c2be9f @generic:default+builtin +[ RUN ] T4 /11ee5e9a @generic:default+builtin +[ OK ] ( 2/10) T4 /11ee5e9a @generic:default+builtin +[ RUN ] T5 /020d01e5 @generic:default+builtin +[ OK ] ( 3/10) T5 /020d01e5 @generic:default+builtin +[ RUN ] T1 /1f93603d @generic:default+builtin +[ OK ] ( 4/10) T1 /1f93603d @generic:default+builtin +[ RUN ] T8 /605fc1d6 @generic:default+builtin +[ FAIL ] ( 5/10) T8 /605fc1d6 @generic:default+builtin +==> test failed during 'setup': test staged in '/home/user/reframe-examples/tutorial/stage/generic/default/builtin/T8' +[ FAIL ] ( 6/10) T9 /78a78a4e @generic:default+builtin ==> test failed during 'startup': test staged in None -[ FAIL ] (12/20) T9 /78a78a4e @tresa:default+clang +[ RUN ] T6 /6dbdaf93 @generic:default+builtin +[ OK ] ( 7/10) T6 /6dbdaf93 @generic:default+builtin +[ RUN ] T2 /0f617ba9 @generic:default+builtin +[ RUN ] T3 /5dd67f7f @generic:default+builtin +[ FAIL ] ( 8/10) T2 /0f617ba9 @generic:default+builtin +==> test failed during 'sanity': test staged in '/home/user/reframe-examples/tutorial/stage/generic/default/builtin/T2' +[ FAIL ] ( 9/10) T7 /f005e93d @generic:default+builtin ==> test failed during 'startup': test staged in None -[ RUN ] T6 /6dbdaf93 @tresa:default+gnu -[ RUN ] T6 /6dbdaf93 @tresa:default+clang -[ OK ] (13/20) T6 /6dbdaf93 @tresa:default+gnu -[ OK ] (14/20) T6 /6dbdaf93 @tresa:default+clang -[ RUN ] T2 /0f617ba9 @tresa:default+gnu -[ RUN ] T2 /0f617ba9 @tresa:default+clang -[ RUN ] T3 /5dd67f7f @tresa:default+gnu -[ RUN ] T3 /5dd67f7f @tresa:default+clang -[ FAIL ] (15/20) T2 /0f617ba9 @tresa:default+gnu -==> test failed during 'sanity': test staged in '/home/user/Repositories/reframe/stage/tresa/default/gnu/T2' -[ FAIL ] (16/20) T2 /0f617ba9 @tresa:default+clang -==> test failed during 'sanity': test staged in '/home/user/Repositories/reframe/stage/tresa/default/clang/T2' -[ FAIL ] (17/20) T7 /f005e93d @tresa:default+gnu -==> test failed during 'startup': test staged in None -[ FAIL ] (18/20) T7 /f005e93d @tresa:default+clang -==> test failed during 'startup': test staged in None -[ OK ] (19/20) T3 /5dd67f7f @tresa:default+gnu -[ OK ] (20/20) T3 /5dd67f7f @tresa:default+clang +[ OK ] (10/10) T3 /5dd67f7f @generic:default+builtin [----------] all spawned checks have finished -[ FAILED ] Ran 20/20 test case(s) from 10 check(s) (8 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:01:03 2022 - +[ FAILED ] Ran 10/10 test case(s) from 10 check(s) (4 failure(s), 0 skipped, 0 aborted) +[==========] Finished on Tue Apr 16 21:35:36 2024+0000 ================================================================================ SUMMARY OF FAILURES -------------------------------------------------------------------------------- -FAILURE INFO for T8 - * Expanded name: T8 +FAILURE INFO for T8 (run: 1/1) * Description: - * System partition: tresa:default - * Environment: gnu - * Stage directory: /home/user/Repositories/reframe/stage/tresa/default/gnu/T8 + * System partition: generic:default + * Environment: builtin + * Stage directory: /home/user/reframe-examples/tutorial/stage/generic/default/builtin/T8 * Node list: * Job type: local (id=None) * Dependencies (conceptual): ['T1'] - * Dependencies (actual): [('T1', 'tresa:default', 'gnu')] + * Dependencies (actual): [('T1', 'generic:default', 'builtin')] * Maintainers: [] * Failing phase: setup - * Rerun with '-n /605fc1d6 -p gnu --system tresa:default -r' + * Rerun with '-n /605fc1d6 -p builtin --system generic:default -r' * Reason: exception Traceback (most recent call last): - File "/home/user/Repositories/reframe/reframe/frontend/executors/__init__.py", line 303, in _safe_call + File "/usr/local/share/reframe/reframe/frontend/executors/__init__.py", line 317, in _safe_call return fn(*args, **kwargs) - File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 101, in _fn + File "/usr/local/share/reframe/reframe/core/hooks.py", line 111, in _fn getattr(obj, h.__name__)() - File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 32, in _fn + File "/usr/local/share/reframe/reframe/core/hooks.py", line 38, in _fn func(*args, **kwargs) - File "/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py", line 180, in fail + File "/home/user/reframe-examples/tutorial/deps/deps_complex.py", line 180, in fail raise Exception Exception -------------------------------------------------------------------------------- -FAILURE INFO for T8 - * Expanded name: T8 - * Description: - * System partition: tresa:default - * Environment: clang - * Stage directory: /home/user/Repositories/reframe/stage/tresa/default/clang/T8 - * Node list: - * Job type: local (id=None) - * Dependencies (conceptual): ['T1'] - * Dependencies (actual): [('T1', 'tresa:default', 'clang')] - * Maintainers: [] - * Failing phase: setup - * Rerun with '-n /605fc1d6 -p clang --system tresa:default -r' - * Reason: exception -Traceback (most recent call last): - File "/home/user/Repositories/reframe/reframe/frontend/executors/__init__.py", line 303, in _safe_call - return fn(*args, **kwargs) - File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 101, in _fn - getattr(obj, h.__name__)() - File "/home/user/Repositories/reframe/reframe/core/hooks.py", line 32, in _fn - func(*args, **kwargs) - File "/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py", line 180, in fail - raise Exception -Exception - --------------------------------------------------------------------------------- -FAILURE INFO for T9 - * Expanded name: T9 - * Description: - * System partition: tresa:default - * Environment: gnu - * Stage directory: None - * Node list: - * Job type: local (id=None) - * Dependencies (conceptual): ['T8'] - * Dependencies (actual): [('T8', 'tresa:default', 'gnu')] - * Maintainers: [] - * Failing phase: startup - * Rerun with '-n /78a78a4e -p gnu --system tresa:default -r' - * Reason: task dependency error: dependencies failed --------------------------------------------------------------------------------- -FAILURE INFO for T9 - * Expanded name: T9 +FAILURE INFO for T9 (run: 1/1) * Description: - * System partition: tresa:default - * Environment: clang + * System partition: generic:default + * Environment: builtin * Stage directory: None * Node list: * Job type: local (id=None) * Dependencies (conceptual): ['T8'] - * Dependencies (actual): [('T8', 'tresa:default', 'clang')] + * Dependencies (actual): [('T8', 'generic:default', 'builtin')] * Maintainers: [] * Failing phase: startup - * Rerun with '-n /78a78a4e -p clang --system tresa:default -r' + * Rerun with '-n /78a78a4e -p builtin --system generic:default -r' * Reason: task dependency error: dependencies failed -------------------------------------------------------------------------------- -FAILURE INFO for T2 - * Expanded name: T2 +FAILURE INFO for T2 (run: 1/1) * Description: - * System partition: tresa:default - * Environment: gnu - * Stage directory: /home/user/Repositories/reframe/stage/tresa/default/gnu/T2 - * Node list: hostNone - * Job type: local (id=59611) + * System partition: generic:default + * Environment: builtin + * Stage directory: /home/user/reframe-examples/tutorial/stage/generic/default/builtin/T2 + * Node list: myhost + * Job type: local (id=23) * Dependencies (conceptual): ['T6'] - * Dependencies (actual): [('T6', 'tresa:default', 'gnu')] + * Dependencies (actual): [('T6', 'generic:default', 'builtin')] * Maintainers: [] * Failing phase: sanity - * Rerun with '-n /0f617ba9 -p gnu --system tresa:default -r' + * Rerun with '-n /0f617ba9 -p builtin --system generic:default -r' * Reason: sanity error: 31 != 30 +--- rfm_job.out (first 10 lines) --- +--- rfm_job.out --- +--- rfm_job.err (first 10 lines) --- +--- rfm_job.err --- -------------------------------------------------------------------------------- -FAILURE INFO for T2 - * Expanded name: T2 - * Description: - * System partition: tresa:default - * Environment: clang - * Stage directory: /home/user/Repositories/reframe/stage/tresa/default/clang/T2 - * Node list: hostNone - * Job type: local (id=59612) - * Dependencies (conceptual): ['T6'] - * Dependencies (actual): [('T6', 'tresa:default', 'clang')] - * Maintainers: [] - * Failing phase: sanity - * Rerun with '-n /0f617ba9 -p clang --system tresa:default -r' - * Reason: sanity error: 31 != 30 --------------------------------------------------------------------------------- -FAILURE INFO for T7 - * Expanded name: T7 - * Description: - * System partition: tresa:default - * Environment: gnu - * Stage directory: None - * Node list: - * Job type: local (id=None) - * Dependencies (conceptual): ['T2'] - * Dependencies (actual): [('T2', 'tresa:default', 'gnu')] - * Maintainers: [] - * Failing phase: startup - * Rerun with '-n /f005e93d -p gnu --system tresa:default -r' - * Reason: task dependency error: dependencies failed --------------------------------------------------------------------------------- -FAILURE INFO for T7 - * Expanded name: T7 +FAILURE INFO for T7 (run: 1/1) * Description: - * System partition: tresa:default - * Environment: clang + * System partition: generic:default + * Environment: builtin * Stage directory: None * Node list: * Job type: local (id=None) * Dependencies (conceptual): ['T2'] - * Dependencies (actual): [('T2', 'tresa:default', 'clang')] + * Dependencies (actual): [('T2', 'generic:default', 'builtin')] * Maintainers: [] * Failing phase: startup - * Rerun with '-n /f005e93d -p clang --system tresa:default -r' + * Rerun with '-n /f005e93d -p builtin --system generic:default -r' * Reason: task dependency error: dependencies failed -------------------------------------------------------------------------------- -Run report saved in '/home/user/.reframe/reports/run-report-326.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-_008n_el.log' +Log file(s) saved in '/tmp/rfm-01gkxmq0.log' diff --git a/docs/listings/deps_rerun_t6.txt b/docs/listings/deps_rerun_t6.txt index 5e7852720f..839d5e8243 100644 --- a/docs/listings/deps_rerun_t6.txt +++ b/docs/listings/deps_rerun_t6.txt @@ -1,25 +1,22 @@ [ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe --restore-session --keep-stage-files -n T6 -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-vtnok1ih.log' + version: 4.6.0-dev.2 + command: '/usr/local/share/reframe/bin/reframe --restore-session --keep-stage-files -n T6 -r --nocolor' + launched by: user@myhost + working directory: '/home/user/reframe-examples/tutorial' + settings files: '' + check search path: '/home/user/reframe-examples/tutorial/deps/deps_complex.py' + stage directory: '/home/user/reframe-examples/tutorial/stage' + output directory: '/home/user/reframe-examples/tutorial/output' + log files: '/tmp/rfm-5nhx1_74.log' [==========] Running 1 check(s) -[==========] Started on Sat Nov 12 19:01:06 2022 +[==========] Started on Wed Apr 3 21:40:44 2024+0000 [----------] start processing checks -[ RUN ] T6 /6dbdaf93 @tresa:default+gnu -[ RUN ] T6 /6dbdaf93 @tresa:default+clang -[ OK ] (1/2) T6 /6dbdaf93 @tresa:default+gnu -[ OK ] (2/2) T6 /6dbdaf93 @tresa:default+clang +[ RUN ] T6 /6dbdaf93 @generic:default+builtin +[ OK ] (1/1) T6 /6dbdaf93 @generic:default+builtin [----------] all spawned checks have finished -[ PASSED ] Ran 2/2 test case(s) from 1 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:01:07 2022 -Run report saved in '/home/user/.reframe/reports/run-report-328.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-vtnok1ih.log' +[ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped, 0 aborted) +[==========] Finished on Wed Apr 3 21:40:44 2024+0000 +Log file(s) saved in '/tmp/rfm-5nhx1_74.log' diff --git a/docs/listings/deps_run_t6.txt b/docs/listings/deps_run_t6.txt index 5a92e562cb..7e07f7cb9c 100644 --- a/docs/listings/deps_run_t6.txt +++ b/docs/listings/deps_run_t6.txt @@ -1,41 +1,30 @@ [ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -n T6 -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/unittests/resources/checks_unlisted/deps_complex.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-8n8uvclh.log' + version: 4.6.0-dev.2 + command: '/usr/local/share/reframe/bin/reframe -c deps/deps_complex.py -n T6 -r --nocolor' + launched by: user@myhost + working directory: '/home/user/reframe-examples/tutorial' + settings files: '' + check search path: '/home/user/reframe-examples/tutorial/deps/deps_complex.py' + stage directory: '/home/user/reframe-examples/tutorial/stage' + output directory: '/home/user/reframe-examples/tutorial/output' + log files: '/tmp/rfm-umx3ijmp.log' [==========] Running 5 check(s) -[==========] Started on Sat Nov 12 19:01:07 2022 +[==========] Started on Wed Apr 3 21:41:17 2024+0000 [----------] start processing checks -[ RUN ] T0 /c9c2be9f @tresa:default+gnu -[ RUN ] T0 /c9c2be9f @tresa:default+clang -[ OK ] ( 1/10) T0 /c9c2be9f @tresa:default+gnu -[ OK ] ( 2/10) T0 /c9c2be9f @tresa:default+clang -[ RUN ] T4 /11ee5e9a @tresa:default+gnu -[ RUN ] T4 /11ee5e9a @tresa:default+clang -[ OK ] ( 3/10) T4 /11ee5e9a @tresa:default+gnu -[ OK ] ( 4/10) T4 /11ee5e9a @tresa:default+clang -[ RUN ] T5 /020d01e5 @tresa:default+gnu -[ RUN ] T5 /020d01e5 @tresa:default+clang -[ OK ] ( 5/10) T5 /020d01e5 @tresa:default+gnu -[ OK ] ( 6/10) T5 /020d01e5 @tresa:default+clang -[ RUN ] T1 /1f93603d @tresa:default+gnu -[ RUN ] T1 /1f93603d @tresa:default+clang -[ OK ] ( 7/10) T1 /1f93603d @tresa:default+gnu -[ OK ] ( 8/10) T1 /1f93603d @tresa:default+clang -[ RUN ] T6 /6dbdaf93 @tresa:default+gnu -[ RUN ] T6 /6dbdaf93 @tresa:default+clang -[ OK ] ( 9/10) T6 /6dbdaf93 @tresa:default+gnu -[ OK ] (10/10) T6 /6dbdaf93 @tresa:default+clang +[ RUN ] T0 /c9c2be9f @generic:default+builtin +[ OK ] (1/5) T0 /c9c2be9f @generic:default+builtin +[ RUN ] T4 /11ee5e9a @generic:default+builtin +[ OK ] (2/5) T4 /11ee5e9a @generic:default+builtin +[ RUN ] T5 /020d01e5 @generic:default+builtin +[ OK ] (3/5) T5 /020d01e5 @generic:default+builtin +[ RUN ] T1 /1f93603d @generic:default+builtin +[ OK ] (4/5) T1 /1f93603d @generic:default+builtin +[ RUN ] T6 /6dbdaf93 @generic:default+builtin +[ OK ] (5/5) T6 /6dbdaf93 @generic:default+builtin [----------] all spawned checks have finished -[ PASSED ] Ran 10/10 test case(s) from 5 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:01:08 2022 -Run report saved in '/home/user/.reframe/reports/run-report-329.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-8n8uvclh.log' +[ PASSED ] Ran 5/5 test case(s) from 5 check(s) (0 failure(s), 0 skipped, 0 aborted) +[==========] Finished on Wed Apr 3 21:41:19 2024+0000 +Log file(s) saved in '/tmp/rfm-umx3ijmp.log' diff --git a/docs/listings/hello1.txt b/docs/listings/hello1.txt deleted file mode 100644 index b7b4c7351d..0000000000 --- a/docs/listings/hello1.txt +++ /dev/null @@ -1,23 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/hello/hello1.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hello/hello1.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-tgqpdq_b.log' - -[==========] Running 1 check(s) -[==========] Started on Sat Nov 12 19:00:44 2022 - -[----------] start processing checks -[ RUN ] HelloTest /2b3e4546 @generic:default+builtin -[ OK ] (1/1) HelloTest /2b3e4546 @generic:default+builtin -[----------] all spawned checks have finished - -[ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:00:45 2022 -Run report saved in '/home/user/.reframe/reports/run-report-319.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-tgqpdq_b.log' diff --git a/docs/listings/hello2.txt b/docs/listings/hello2.txt deleted file mode 100644 index c7b0b0701f..0000000000 --- a/docs/listings/hello2.txt +++ /dev/null @@ -1,46 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/hello/hello2.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-krmo7oc3.log' - -[==========] Running 2 check(s) -[==========] Started on Sat Nov 12 19:00:45 2022 - -[----------] start processing checks -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @generic:default+builtin -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @generic:default+builtin -[ FAIL ] (1/2) HelloMultiLangTest %lang=cpp /71bf65a3 @generic:default+builtin -==> test failed during 'compile': test staged in '/home/user/Repositories/reframe/stage/generic/default/builtin/HelloMultiLangTest_71bf65a3' -rfm_job.out -[ OK ] (2/2) HelloMultiLangTest %lang=c /7cfa870e @generic:default+builtin -[----------] all spawned checks have finished - -[ FAILED ] Ran 2/2 test case(s) from 2 check(s) (1 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:00:46 2022 - -================================================================================ -SUMMARY OF FAILURES --------------------------------------------------------------------------------- -FAILURE INFO for HelloMultiLangTest_1 - * Expanded name: HelloMultiLangTest %lang=cpp - * Description: - * System partition: generic:default - * Environment: builtin - * Stage directory: /home/user/Repositories/reframe/stage/generic/default/builtin/HelloMultiLangTest_71bf65a3 - * Node list: - * Job type: local (id=None) - * Dependencies (conceptual): [] - * Dependencies (actual): [] - * Maintainers: [] - * Failing phase: compile - * Rerun with '-n /71bf65a3 -p builtin --system generic:default -r' - * Reason: build system error: I do not know how to compile a C++ program --------------------------------------------------------------------------------- -Run report saved in '/home/user/.reframe/reports/run-report-320.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-krmo7oc3.log' diff --git a/docs/listings/hello2_list_verbose.txt b/docs/listings/hello2_list_verbose.txt deleted file mode 100644 index ba65ade6a1..0000000000 --- a/docs/listings/hello2_list_verbose.txt +++ /dev/null @@ -1,100 +0,0 @@ -Loading user configuration -Loading the generic configuration -Loading configuration file: ('tutorials/config/tresa.py',) -Detecting system using method: 'hostname' -Using standard hostname... -Retrieved hostname: 'host' -Looking for a matching configuration entry -Configuration found: picking system 'tresa' -Initializing runtime -Initializing system partition 'default' -Initializing system 'tresa' -Initializing modules system 'nomod' -detecting topology info for tresa:default -> found topology file '/home/user/.reframe/topology/tresa-default/processor.json'; loading... -> device auto-detection is not supported -[ReFrame Environment] - RFM_AUTODETECT_FQDN= - RFM_AUTODETECT_METHOD= - RFM_AUTODETECT_XTHOSTNAME= - RFM_CHECK_SEARCH_PATH= - RFM_CHECK_SEARCH_RECURSIVE= - RFM_CLEAN_STAGEDIR= - RFM_COLORIZE=n - RFM_COMPRESS_REPORT= - RFM_CONFIG_FILES=/home/user/Repositories/reframe/tutorials/config/tresa.py - RFM_CONFIG_PATH= - RFM_DUMP_PIPELINE_PROGRESS= - RFM_GIT_TIMEOUT= - RFM_HTTPJSON_URL= - RFM_IGNORE_CHECK_CONFLICTS= - RFM_IGNORE_REQNODENOTAVAIL= - RFM_INSTALL_PREFIX=/home/user/Repositories/reframe - RFM_KEEP_STAGE_FILES= - RFM_MODULE_MAPPINGS= - RFM_MODULE_MAP_FILE= - RFM_NON_DEFAULT_CRAYPE= - RFM_OUTPUT_DIR= - RFM_PERFLOG_DIR= - RFM_PIPELINE_TIMEOUT= - RFM_PREFIX= - RFM_PURGE_ENVIRONMENT= - RFM_REMOTE_DETECT= - RFM_REMOTE_WORKDIR= - RFM_REPORT_FILE= - RFM_REPORT_JUNIT= - RFM_RESOLVE_MODULE_CONFLICTS= - RFM_SAVE_LOG_FILES= - RFM_STAGE_DIR= - RFM_SYSLOG_ADDRESS= - RFM_SYSTEM= - RFM_TIMESTAMP_DIRS= - RFM_TRAP_JOB_ERRORS= - RFM_UNLOAD_MODULES= - RFM_USER_MODULES= - RFM_USE_LOGIN_SHELL= - RFM_VERBOSE= -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -C tutorials/config/tresa.py -c tutorials/basics/hello/hello2.py -l -vv' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', 'tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-3gcehyof.log' - -Looking for tests in '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py' -Validating '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py': OK - > Loaded 2 test(s) -Loaded 2 test(s) -Generated 4 test case(s) -Filtering test cases(s) by name: 4 remaining -Filtering test cases(s) by tags: 4 remaining -Filtering test cases(s) by other attributes: 4 remaining -Building and validating the full test DAG -Full test DAG: - ('HelloMultiLangTest_1', 'tresa:default', 'gnu') -> [] - ('HelloMultiLangTest_1', 'tresa:default', 'clang') -> [] - ('HelloMultiLangTest_0', 'tresa:default', 'gnu') -> [] - ('HelloMultiLangTest_0', 'tresa:default', 'clang') -> [] -Final number of test cases: 4 -[List of matched checks] -- HelloMultiLangTest %lang=cpp /71bf65a3 -- HelloMultiLangTest %lang=c /7cfa870e -Found 2 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-3gcehyof.log' ->>> profiler report [start] <<< -main: 0.053832 s - test processing: 0.012268 s - RegressionCheckLoader.load_all: 0.008720 s - TestRegistry.instantiate_all: 0.003012 s - generate_testcases: 0.000049 s - main.._sort_testcases: 0.000012 s - build_deps: 0.000072 s - validate_deps: 0.000061 s - toposort: 0.000091 s - list_checks: 0.001080 s ->>> profiler report [ end ] <<< diff --git a/docs/listings/hello2_print_stdout.txt b/docs/listings/hello2_print_stdout.txt deleted file mode 100644 index ada4bd4c2a..0000000000 --- a/docs/listings/hello2_print_stdout.txt +++ /dev/null @@ -1,37 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -C tutorials/config/tresa.py -c tutorials/basics/hello/hello2.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', 'tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-b22mnhb0.log' - -[==========] Running 2 check(s) -[==========] Started on Sat Nov 12 19:00:58 2022 - -[----------] start processing checks -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+gnu -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+clang -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @tresa:default+gnu -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @tresa:default+clang -rfm_job.out -rfm_job.out -[ OK ] (1/4) HelloMultiLangTest %lang=c /7cfa870e @tresa:default+gnu -rfm_job.out -rfm_job.out -[ OK ] (2/4) HelloMultiLangTest %lang=c /7cfa870e @tresa:default+clang -rfm_job.out -rfm_job.out -[ OK ] (3/4) HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+gnu -rfm_job.out -rfm_job.out -[ OK ] (4/4) HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+clang -[----------] all spawned checks have finished - -[ PASSED ] Ran 4/4 test case(s) from 2 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:01:00 2022 -Run report saved in '/home/user/.reframe/reports/run-report-325.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-b22mnhb0.log' diff --git a/docs/listings/hello2_tresa.txt b/docs/listings/hello2_tresa.txt deleted file mode 100644 index a594563a79..0000000000 --- a/docs/listings/hello2_tresa.txt +++ /dev/null @@ -1,33 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -C tutorials/config/tresa.py -c tutorials/basics/hello/hello2.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', 'tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-e3dlf19_.log' - -[==========] Running 2 check(s) -[==========] Started on Sat Nov 12 19:00:46 2022 - -[----------] start processing checks -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+gnu -[ RUN ] HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+clang -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @tresa:default+gnu -[ RUN ] HelloMultiLangTest %lang=c /7cfa870e @tresa:default+clang -rfm_job.out -[ OK ] (1/4) HelloMultiLangTest %lang=c /7cfa870e @tresa:default+gnu -rfm_job.out -[ OK ] (2/4) HelloMultiLangTest %lang=c /7cfa870e @tresa:default+clang -rfm_job.out -[ OK ] (3/4) HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+gnu -rfm_job.out -[ OK ] (4/4) HelloMultiLangTest %lang=cpp /71bf65a3 @tresa:default+clang -[----------] all spawned checks have finished - -[ PASSED ] Ran 4/4 test case(s) from 2 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:00:48 2022 -Run report saved in '/home/user/.reframe/reports/run-report-321.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-e3dlf19_.log' diff --git a/docs/listings/hello2_typo.txt b/docs/listings/hello2_typo.txt deleted file mode 100644 index 8446dc8256..0000000000 --- a/docs/listings/hello2_typo.txt +++ /dev/null @@ -1,19 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/hello -R -l' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: (R) '/home/user/Repositories/reframe/tutorials/basics/hello' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-ldo5um3v.log' - -WARNING: skipping test file '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py': name error: tutorials/basics/hello/hello2.py:13: name 'paramter' is not defined - lang = paramter(['c', 'cpp']) - (rerun with '-v' for more information) -[List of matched checks] -- HelloTest /2b3e4546 -Found 1 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-ldo5um3v.log' diff --git a/docs/listings/hello2_typo_stacktrace.txt b/docs/listings/hello2_typo_stacktrace.txt deleted file mode 100644 index 8df6d423e8..0000000000 --- a/docs/listings/hello2_typo_stacktrace.txt +++ /dev/null @@ -1,44 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/hello -R -l -v' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: (R) '/home/user/Repositories/reframe/tutorials/basics/hello' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-xs3l6jud.log' - -WARNING: skipping test file '/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py': name error: tutorials/basics/hello/hello2.py:13: name 'paramter' is not defined - lang = paramter(['c', 'cpp']) - (rerun with '-v' for more information) -Traceback (most recent call last): - File "/home/user/Repositories/reframe/reframe/frontend/loader.py", line 205, in load_from_file - util.import_module_from_file(filename, force) - File "/home/user/Repositories/reframe/reframe/utility/__init__.py", line 109, in import_module_from_file - return importlib.import_module(module_name) - File "/usr/local/Cellar/python@3.10/3.10.7/Frameworks/Python.framework/Versions/3.10/lib/python3.10/importlib/__init__.py", line 126, in import_module - return _bootstrap._gcd_import(name[level:], package, level) - File "", line 1050, in _gcd_import - File "", line 1027, in _find_and_load - File "", line 1006, in _find_and_load_unlocked - File "", line 688, in _load_unlocked - File "", line 883, in exec_module - File "", line 241, in _call_with_frames_removed - File "/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py", line 12, in - class HelloMultiLangTest(rfm.RegressionTest): - File "/home/user/Repositories/reframe/tutorials/basics/hello/hello2.py", line 13, in HelloMultiLangTest - lang = paramter(['c', 'cpp']) -NameError: name 'paramter' is not defined - -Loaded 1 test(s) -Generated 2 test case(s) -Filtering test cases(s) by name: 2 remaining -Filtering test cases(s) by tags: 2 remaining -Filtering test cases(s) by other attributes: 2 remaining -Final number of test cases: 2 -[List of matched checks] -- HelloTest /2b3e4546 -Found 1 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-xs3l6jud.log' diff --git a/docs/listings/hello2_verbose_load.txt b/docs/listings/hello2_verbose_load.txt deleted file mode 100644 index b97ee20a93..0000000000 --- a/docs/listings/hello2_verbose_load.txt +++ /dev/null @@ -1,80 +0,0 @@ -Loading user configuration -Loading configuration file: 'tutorials/config/settings.py' -Detecting system -Looking for a matching configuration entry for system '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa' -Configuration found: picking system 'generic' -Selecting subconfig for 'generic' -Initializing runtime -Selecting subconfig for 'generic:default' -Initializing system partition 'default' -Selecting subconfig for 'generic' -Initializing system 'generic' -Initializing modules system 'nomod' -detecting topology info for generic:default -> found topology file '/Users/user/.reframe/topology/generic-default/processor.json'; loading... -> device auto-detection is not supported -[ReFrame Environment] - RFM_CHECK_SEARCH_PATH= - RFM_CHECK_SEARCH_RECURSIVE= - RFM_CLEAN_STAGEDIR= - RFM_COLORIZE= - RFM_COMPACT_TEST_NAMES= - RFM_CONFIG_FILE=tutorials/config/settings.py - RFM_GIT_TIMEOUT= - RFM_GRAYLOG_ADDRESS= - RFM_HTTPJSON_URL= - RFM_IGNORE_CHECK_CONFLICTS= - RFM_IGNORE_REQNODENOTAVAIL= - RFM_INSTALL_PREFIX=/Users/user/Repositories/reframe - RFM_KEEP_STAGE_FILES= - RFM_MODULE_MAPPINGS= - RFM_MODULE_MAP_FILE= - RFM_NON_DEFAULT_CRAYPE= - RFM_OUTPUT_DIR= - RFM_PERFLOG_DIR= - RFM_PREFIX= - RFM_PURGE_ENVIRONMENT= - RFM_REMOTE_DETECT= - RFM_REMOTE_WORKDIR= - RFM_REPORT_FILE= - RFM_REPORT_JUNIT= - RFM_RESOLVE_MODULE_CONFLICTS= - RFM_SAVE_LOG_FILES= - RFM_STAGE_DIR= - RFM_SYSLOG_ADDRESS= - RFM_SYSTEM= - RFM_TIMESTAMP_DIRS= - RFM_TRAP_JOB_ERRORS= - RFM_UNLOAD_MODULES= - RFM_USER_MODULES= - RFM_USE_LOGIN_SHELL= - RFM_VERBOSE= -[ReFrame Setup] - version: 3.10.0-dev.2+cb5edd8b - command: './bin/reframe -C tutorials/config/settings.py -c tutorials/basics/hello/hello2.py -l -vv' - launched by: user@host - working directory: '/Users/user/Repositories/reframe' - settings file: 'tutorials/config/settings.py' - check search path: '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py' - stage directory: '/Users/user/Repositories/reframe/stage' - output directory: '/Users/user/Repositories/reframe/output' - -Looking for tests in '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py' -Validating '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py': OK - > Loaded 2 test(s) -Loaded 2 test(s) -Generated 2 test case(s) -Filtering test cases(s) by name: 2 remaining -Filtering test cases(s) by tags: 2 remaining -Filtering test cases(s) by other attributes: 2 remaining -Building and validating the full test DAG -Full test DAG: - ('HelloMultiLangTest_cpp', 'generic:default', 'builtin') -> [] - ('HelloMultiLangTest_c', 'generic:default', 'builtin') -> [] -Final number of test cases: 2 -[List of matched checks] -- HelloMultiLangTest %lang=cpp -- HelloMultiLangTest %lang=c -Found 2 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-fpjj5gru.log' diff --git a/docs/listings/hellomp1.txt b/docs/listings/hellomp1.txt deleted file mode 100644 index 0dd52677ee..0000000000 --- a/docs/listings/hellomp1.txt +++ /dev/null @@ -1,25 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/hellomp/hellomp1.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hellomp/hellomp1.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-v56bz2uo.log' - -[==========] Running 1 check(s) -[==========] Started on Sat Nov 12 19:00:48 2022 - -[----------] start processing checks -[ RUN ] HelloThreadedTest /a6fa300f @tresa:default+gnu -[ RUN ] HelloThreadedTest /a6fa300f @tresa:default+clang -[ OK ] (1/2) HelloThreadedTest /a6fa300f @tresa:default+gnu -[ OK ] (2/2) HelloThreadedTest /a6fa300f @tresa:default+clang -[----------] all spawned checks have finished - -[ PASSED ] Ran 2/2 test case(s) from 1 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:00:50 2022 -Run report saved in '/home/user/.reframe/reports/run-report-322.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-v56bz2uo.log' diff --git a/docs/listings/hellomp2.txt b/docs/listings/hellomp2.txt deleted file mode 100644 index 68072e954d..0000000000 --- a/docs/listings/hellomp2.txt +++ /dev/null @@ -1,61 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/hellomp/hellomp2.py -r' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/basics/hellomp/hellomp2.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-a2tt4eqp.log' - -[==========] Running 1 check(s) -[==========] Started on Sat Nov 12 19:00:50 2022 - -[----------] start processing checks -[ RUN ] HelloThreadedExtendedTest /4733a67d @tresa:default+gnu -[ RUN ] HelloThreadedExtendedTest /4733a67d @tresa:default+clang -[ FAIL ] (1/2) HelloThreadedExtendedTest /4733a67d @tresa:default+gnu -==> test failed during 'sanity': test staged in '/home/user/Repositories/reframe/stage/tresa/default/gnu/HelloThreadedExtendedTest' -[ FAIL ] (2/2) HelloThreadedExtendedTest /4733a67d @tresa:default+clang -==> test failed during 'sanity': test staged in '/home/user/Repositories/reframe/stage/tresa/default/clang/HelloThreadedExtendedTest' -[----------] all spawned checks have finished - -[ FAILED ] Ran 2/2 test case(s) from 1 check(s) (2 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:00:52 2022 - -================================================================================ -SUMMARY OF FAILURES --------------------------------------------------------------------------------- -FAILURE INFO for HelloThreadedExtendedTest - * Expanded name: HelloThreadedExtendedTest - * Description: - * System partition: tresa:default - * Environment: gnu - * Stage directory: /home/user/Repositories/reframe/stage/tresa/default/gnu/HelloThreadedExtendedTest - * Node list: hostNone - * Job type: local (id=59525) - * Dependencies (conceptual): [] - * Dependencies (actual): [] - * Maintainers: [] - * Failing phase: sanity - * Rerun with '-n /4733a67d -p gnu --system tresa:default -r' - * Reason: sanity error: 13 != 16 --------------------------------------------------------------------------------- -FAILURE INFO for HelloThreadedExtendedTest - * Expanded name: HelloThreadedExtendedTest - * Description: - * System partition: tresa:default - * Environment: clang - * Stage directory: /home/user/Repositories/reframe/stage/tresa/default/clang/HelloThreadedExtendedTest - * Node list: hostNone - * Job type: local (id=59528) - * Dependencies (conceptual): [] - * Dependencies (actual): [] - * Maintainers: [] - * Failing phase: sanity - * Rerun with '-n /4733a67d -p clang --system tresa:default -r' - * Reason: sanity error: 11 != 16 --------------------------------------------------------------------------------- -Run report saved in '/home/user/.reframe/reports/run-report-323.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-a2tt4eqp.log' diff --git a/docs/listings/maketest_mixin.txt b/docs/listings/maketest_mixin.txt deleted file mode 100644 index 039fc0cc21..0000000000 --- a/docs/listings/maketest_mixin.txt +++ /dev/null @@ -1,19 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/advanced/makefiles/maketest_mixin.py -l' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/advanced/makefiles/maketest_mixin.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-z_z51hkz.log' - -[List of matched checks] -- MakeOnlyTestAlt %elem_type=double /8b62380e -- MakeOnlyTestAlt %elem_type=float /da39ec20 -- MakefileTestAlt %elem_type=double /89aac4a2 -- MakefileTestAlt %elem_type=float /a998ce67 -Found 4 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-z_z51hkz.log' diff --git a/docs/listings/osu_bandwidth_concretized_daint.txt b/docs/listings/osu_bandwidth_concretized_daint.txt deleted file mode 100644 index d16b17007b..0000000000 --- a/docs/listings/osu_bandwidth_concretized_daint.txt +++ /dev/null @@ -1,24 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/fixtures/osu_benchmarks.py -n osu_bandwidth_test -lC' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-m1w2t4eh.log' - -[List of matched checks] -- osu_bandwidth_test /026711a1 @daint:gpu+gnu - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 @daint:gpu+gnu - ^fetch_osu_benchmarks ~daint /79cd6023 @daint:gpu+gnu -- osu_bandwidth_test /026711a1 @daint:gpu+intel - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 @daint:gpu+intel - ^fetch_osu_benchmarks ~daint /79cd6023 @daint:gpu+gnu -- osu_bandwidth_test /026711a1 @daint:gpu+nvidia - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 @daint:gpu+nvidia - ^fetch_osu_benchmarks ~daint /79cd6023 @daint:gpu+gnu -Concretized 7 test case(s) - -Log file(s) saved in '/tmp/rfm-m1w2t4eh.log' diff --git a/docs/listings/osu_bandwidth_concretized_daint_nvidia.txt b/docs/listings/osu_bandwidth_concretized_daint_nvidia.txt deleted file mode 100644 index 7ccdba96a9..0000000000 --- a/docs/listings/osu_bandwidth_concretized_daint_nvidia.txt +++ /dev/null @@ -1,17 +0,0 @@ -[ReFrame Setup] - version: 3.10.0-dev.3+605af31a - command: './bin/reframe -c tutorials/fixtures/osu_benchmarks.py -n osu_bandwidth_test -lC -p nvidia' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings file: '/home/user/Devel/reframe/tutorials/config/settings.py' - check search path: '/home/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - -[List of matched checks] -- osu_bandwidth_test @daint:gpu+nvidia - ^build_osu_benchmarks ~daint:gpu+nvidia @daint:gpu+nvidia - ^fetch_osu_benchmarks ~daint @daint:gpu+nvidia -Concretized 3 test case(s) - -Log file(s) saved in '/tmp/rfm-dnfdagj8.log' diff --git a/docs/listings/osu_bench_deps.txt b/docs/listings/osu_bench_deps.txt deleted file mode 100644 index 974d410d8f..0000000000 --- a/docs/listings/osu_bench_deps.txt +++ /dev/null @@ -1,83 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/deps/osu_benchmarks.py -r' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-r1a7v0w3.log' - -[==========] Running 8 check(s) -[==========] Started on Tue Nov 15 18:24:00 2022 - -[----------] start processing checks -[ RUN ] OSUDownloadTest /7de668df @daint:login+builtin -[ OK ] ( 1/22) OSUDownloadTest /7de668df @daint:login+builtin -[ RUN ] OSUBuildTest /19b4fb56 @daint:gpu+gnu -[ RUN ] OSUBuildTest /19b4fb56 @daint:gpu+intel -[ RUN ] OSUBuildTest /19b4fb56 @daint:gpu+nvidia -[ OK ] ( 2/22) OSUBuildTest /19b4fb56 @daint:gpu+gnu -[ RUN ] OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+gnu -[ RUN ] OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+gnu -[ RUN ] OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+gnu -[ RUN ] OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+gnu -[ RUN ] OSUBandwidthTest /764cdb0b @daint:gpu+gnu -[ RUN ] OSULatencyTest /14f35a43 @daint:gpu+gnu -[ OK ] ( 3/22) OSUBuildTest /19b4fb56 @daint:gpu+intel -[ RUN ] OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+intel -[ RUN ] OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+intel -[ RUN ] OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+intel -[ OK ] ( 4/22) OSUBuildTest /19b4fb56 @daint:gpu+nvidia -[ RUN ] OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+nvidia -[ RUN ] OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+nvidia -[ RUN ] OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+nvidia -[ RUN ] OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+intel -[ RUN ] OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+nvidia -[ RUN ] OSUBandwidthTest /764cdb0b @daint:gpu+intel -[ RUN ] OSUBandwidthTest /764cdb0b @daint:gpu+nvidia -[ RUN ] OSULatencyTest /14f35a43 @daint:gpu+intel -[ RUN ] OSULatencyTest /14f35a43 @daint:gpu+nvidia -[ OK ] ( 5/22) OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+gnu -P: latency: 5.31 us (r:0, l:None, u:None) -[ OK ] ( 6/22) OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+intel -P: latency: 10.07 us (r:0, l:None, u:None) -[ OK ] ( 7/22) OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+gnu -P: latency: 1.67 us (r:0, l:None, u:None) -[ OK ] ( 8/22) OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+intel -P: latency: 24.97 us (r:0, l:None, u:None) -[ OK ] ( 9/22) OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+nvidia -P: latency: 8.92 us (r:0, l:None, u:None) -[ OK ] (10/22) OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+intel -P: latency: 14.78 us (r:0, l:None, u:None) -[ OK ] (11/22) OSULatencyTest /14f35a43 @daint:gpu+nvidia -P: latency: 2.19 us (r:0, l:None, u:None) -[ OK ] (12/22) OSULatencyTest /14f35a43 @daint:gpu+gnu -P: latency: 1.76 us (r:0, l:None, u:None) -[ OK ] (13/22) OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+gnu -P: latency: 19.54 us (r:0, l:None, u:None) -[ OK ] (14/22) OSULatencyTest /14f35a43 @daint:gpu+intel -P: latency: 4.4 us (r:0, l:None, u:None) -[ OK ] (15/22) OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+nvidia -P: latency: 6.88 us (r:0, l:None, u:None) -[ OK ] (16/22) OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+intel -P: latency: 21.37 us (r:0, l:None, u:None) -[ OK ] (17/22) OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+gnu -P: latency: 10.15 us (r:0, l:None, u:None) -[ OK ] (18/22) OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+nvidia -P: latency: 52.87 us (r:0, l:None, u:None) -[ OK ] (19/22) OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+nvidia -P: latency: 64.77 us (r:0, l:None, u:None) -[ OK ] (20/22) OSUBandwidthTest /764cdb0b @daint:gpu+intel -P: bandwidth: 9118.51 MB/s (r:0, l:None, u:None) -[ OK ] (21/22) OSUBandwidthTest /764cdb0b @daint:gpu+nvidia -P: bandwidth: 8476.18 MB/s (r:0, l:None, u:None) -[ OK ] (22/22) OSUBandwidthTest /764cdb0b @daint:gpu+gnu -P: bandwidth: 8326.06 MB/s (r:0, l:None, u:None) -[----------] all spawned checks have finished - -[ PASSED ] Ran 22/22 test case(s) from 8 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Tue Nov 15 18:27:13 2022 -Run report saved in '/home/user/.reframe/reports/run-report-3.json' -Log file(s) saved in '/tmp/rfm-r1a7v0w3.log' diff --git a/docs/listings/osu_bench_fixtures_list.txt b/docs/listings/osu_bench_fixtures_list.txt deleted file mode 100644 index 19fbd22500..0000000000 --- a/docs/listings/osu_bench_fixtures_list.txt +++ /dev/null @@ -1,57 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/fixtures/osu_benchmarks.py -l' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-to7wa4gh.log' - -[List of matched checks] -- osu_allreduce_test %mpi_tasks=16 /1fe48834 - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 - ^fetch_osu_benchmarks ~daint /79cd6023 -- osu_allreduce_test %mpi_tasks=8 /ae01c137 - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 - ^fetch_osu_benchmarks ~daint /79cd6023 -- osu_allreduce_test %mpi_tasks=4 /2129dc34 - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 - ^fetch_osu_benchmarks ~daint /79cd6023 -- osu_allreduce_test %mpi_tasks=2 /9f29c081 - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 - ^fetch_osu_benchmarks ~daint /79cd6023 -- osu_bandwidth_test /026711a1 - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 - ^fetch_osu_benchmarks ~daint /79cd6023 -- osu_latency_test /d2c978ad - ^build_osu_benchmarks ~daint:gpu+gnu /f3269d42 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+intel /4d450880 - ^fetch_osu_benchmarks ~daint /79cd6023 - ^build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 - ^fetch_osu_benchmarks ~daint /79cd6023 -Found 6 check(s) - -Log file(s) saved in '/tmp/rfm-to7wa4gh.log' diff --git a/docs/listings/osu_bench_fixtures_run.txt b/docs/listings/osu_bench_fixtures_run.txt deleted file mode 100644 index 401ef37227..0000000000 --- a/docs/listings/osu_bench_fixtures_run.txt +++ /dev/null @@ -1,83 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/fixtures/osu_benchmarks.py -r' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-63lwmv4b.log' - -[==========] Running 10 check(s) -[==========] Started on Tue Nov 15 18:27:17 2022 - -[----------] start processing checks -[ RUN ] fetch_osu_benchmarks ~daint /79cd6023 @daint:gpu+gnu -[ OK ] ( 1/22) fetch_osu_benchmarks ~daint /79cd6023 @daint:gpu+gnu -[ RUN ] build_osu_benchmarks ~daint:gpu+gnu /f3269d42 @daint:gpu+gnu -[ RUN ] build_osu_benchmarks ~daint:gpu+intel /4d450880 @daint:gpu+intel -[ RUN ] build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 @daint:gpu+nvidia -[ OK ] ( 2/22) build_osu_benchmarks ~daint:gpu+gnu /f3269d42 @daint:gpu+gnu -[ RUN ] osu_allreduce_test %mpi_tasks=16 /1fe48834 @daint:gpu+gnu -[ RUN ] osu_allreduce_test %mpi_tasks=8 /ae01c137 @daint:gpu+gnu -[ RUN ] osu_allreduce_test %mpi_tasks=4 /2129dc34 @daint:gpu+gnu -[ RUN ] osu_allreduce_test %mpi_tasks=2 /9f29c081 @daint:gpu+gnu -[ RUN ] osu_bandwidth_test /026711a1 @daint:gpu+gnu -[ RUN ] osu_latency_test /d2c978ad @daint:gpu+gnu -[ OK ] ( 3/22) build_osu_benchmarks ~daint:gpu+intel /4d450880 @daint:gpu+intel -[ OK ] ( 4/22) build_osu_benchmarks ~daint:gpu+nvidia /e9b8d152 @daint:gpu+nvidia -[ RUN ] osu_allreduce_test %mpi_tasks=16 /1fe48834 @daint:gpu+intel -[ RUN ] osu_allreduce_test %mpi_tasks=16 /1fe48834 @daint:gpu+nvidia -[ RUN ] osu_allreduce_test %mpi_tasks=8 /ae01c137 @daint:gpu+intel -[ RUN ] osu_allreduce_test %mpi_tasks=8 /ae01c137 @daint:gpu+nvidia -[ RUN ] osu_allreduce_test %mpi_tasks=4 /2129dc34 @daint:gpu+intel -[ RUN ] osu_allreduce_test %mpi_tasks=4 /2129dc34 @daint:gpu+nvidia -[ RUN ] osu_allreduce_test %mpi_tasks=2 /9f29c081 @daint:gpu+intel -[ RUN ] osu_allreduce_test %mpi_tasks=2 /9f29c081 @daint:gpu+nvidia -[ RUN ] osu_bandwidth_test /026711a1 @daint:gpu+intel -[ RUN ] osu_bandwidth_test /026711a1 @daint:gpu+nvidia -[ RUN ] osu_latency_test /d2c978ad @daint:gpu+intel -[ RUN ] osu_latency_test /d2c978ad @daint:gpu+nvidia -[ OK ] ( 5/22) osu_allreduce_test %mpi_tasks=2 /9f29c081 @daint:gpu+gnu -P: latency: 2.76 us (r:0, l:None, u:None) -[ OK ] ( 6/22) osu_allreduce_test %mpi_tasks=2 /9f29c081 @daint:gpu+intel -P: latency: 1.68 us (r:0, l:None, u:None) -[ OK ] ( 7/22) osu_allreduce_test %mpi_tasks=4 /2129dc34 @daint:gpu+intel -P: latency: 4.89 us (r:0, l:None, u:None) -[ OK ] ( 8/22) osu_latency_test /d2c978ad @daint:gpu+intel -P: latency: 1.54 us (r:0, l:None, u:None) -[ OK ] ( 9/22) osu_latency_test /d2c978ad @daint:gpu+gnu -P: latency: 1.17 us (r:0, l:None, u:None) -[ OK ] (10/22) osu_allreduce_test %mpi_tasks=4 /2129dc34 @daint:gpu+gnu -P: latency: 3.22 us (r:0, l:None, u:None) -[ OK ] (11/22) osu_allreduce_test %mpi_tasks=16 /1fe48834 @daint:gpu+gnu -P: latency: 13.84 us (r:0, l:None, u:None) -[ OK ] (12/22) osu_allreduce_test %mpi_tasks=16 /1fe48834 @daint:gpu+nvidia -P: latency: 30.77 us (r:0, l:None, u:None) -[ OK ] (13/22) osu_allreduce_test %mpi_tasks=4 /2129dc34 @daint:gpu+nvidia -P: latency: 5.74 us (r:0, l:None, u:None) -[ OK ] (14/22) osu_allreduce_test %mpi_tasks=16 /1fe48834 @daint:gpu+intel -P: latency: 14.77 us (r:0, l:None, u:None) -[ OK ] (15/22) osu_allreduce_test %mpi_tasks=2 /9f29c081 @daint:gpu+nvidia -P: latency: 4.5 us (r:0, l:None, u:None) -[ OK ] (16/22) osu_allreduce_test %mpi_tasks=8 /ae01c137 @daint:gpu+nvidia -P: latency: 33.93 us (r:0, l:None, u:None) -[ OK ] (17/22) osu_allreduce_test %mpi_tasks=8 /ae01c137 @daint:gpu+intel -P: latency: 20.9 us (r:0, l:None, u:None) -[ OK ] (18/22) osu_latency_test /d2c978ad @daint:gpu+nvidia -P: latency: 1.18 us (r:0, l:None, u:None) -[ OK ] (19/22) osu_allreduce_test %mpi_tasks=8 /ae01c137 @daint:gpu+gnu -P: latency: 10.14 us (r:0, l:None, u:None) -[ OK ] (20/22) osu_bandwidth_test /026711a1 @daint:gpu+gnu -P: bandwidth: 9785.43 MB/s (r:0, l:None, u:None) -[ OK ] (21/22) osu_bandwidth_test /026711a1 @daint:gpu+intel -P: bandwidth: 9841.26 MB/s (r:0, l:None, u:None) -[ OK ] (22/22) osu_bandwidth_test /026711a1 @daint:gpu+nvidia -P: bandwidth: 9824.01 MB/s (r:0, l:None, u:None) -[----------] all spawned checks have finished - -[ PASSED ] Ran 22/22 test case(s) from 10 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Tue Nov 15 18:30:34 2022 -Run report saved in '/home/user/.reframe/reports/run-report-4.json' -Log file(s) saved in '/tmp/rfm-63lwmv4b.log' diff --git a/docs/listings/osu_bench_list_concretized.txt b/docs/listings/osu_bench_list_concretized.txt deleted file mode 100644 index 598a42906a..0000000000 --- a/docs/listings/osu_bench_list_concretized.txt +++ /dev/null @@ -1,69 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/deps/osu_benchmarks.py -lC' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-fremrbwf.log' - -[List of matched checks] -- OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+gnu - ^OSUBuildTest /19b4fb56 @daint:gpu+gnu - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+intel - ^OSUBuildTest /19b4fb56 @daint:gpu+intel - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=16 /7f033d39 @daint:gpu+nvidia - ^OSUBuildTest /19b4fb56 @daint:gpu+nvidia - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+gnu - ^OSUBuildTest /19b4fb56 @daint:gpu+gnu - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+intel - ^OSUBuildTest /19b4fb56 @daint:gpu+intel - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=8 /005fca19 @daint:gpu+nvidia - ^OSUBuildTest /19b4fb56 @daint:gpu+nvidia - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+gnu - ^OSUBuildTest /19b4fb56 @daint:gpu+gnu - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+intel - ^OSUBuildTest /19b4fb56 @daint:gpu+intel - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=4 /84b85d90 @daint:gpu+nvidia - ^OSUBuildTest /19b4fb56 @daint:gpu+nvidia - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+gnu - ^OSUBuildTest /19b4fb56 @daint:gpu+gnu - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+intel - ^OSUBuildTest /19b4fb56 @daint:gpu+intel - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUAllreduceTest %mpi_tasks=2 /9d550c4f @daint:gpu+nvidia - ^OSUBuildTest /19b4fb56 @daint:gpu+nvidia - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUBandwidthTest /764cdb0b @daint:gpu+gnu - ^OSUBuildTest /19b4fb56 @daint:gpu+gnu - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUBandwidthTest /764cdb0b @daint:gpu+intel - ^OSUBuildTest /19b4fb56 @daint:gpu+intel - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSUBandwidthTest /764cdb0b @daint:gpu+nvidia - ^OSUBuildTest /19b4fb56 @daint:gpu+nvidia - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSULatencyTest /14f35a43 @daint:gpu+gnu - ^OSUBuildTest /19b4fb56 @daint:gpu+gnu - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSULatencyTest /14f35a43 @daint:gpu+intel - ^OSUBuildTest /19b4fb56 @daint:gpu+intel - ^OSUDownloadTest /7de668df @daint:login+builtin -- OSULatencyTest /14f35a43 @daint:gpu+nvidia - ^OSUBuildTest /19b4fb56 @daint:gpu+nvidia - ^OSUDownloadTest /7de668df @daint:login+builtin -Concretized 22 test case(s) - -Log file(s) saved in '/tmp/rfm-fremrbwf.log' diff --git a/docs/listings/osu_bench_list_concretized_gnu.txt b/docs/listings/osu_bench_list_concretized_gnu.txt deleted file mode 100644 index 7898775b1d..0000000000 --- a/docs/listings/osu_bench_list_concretized_gnu.txt +++ /dev/null @@ -1,18 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/deps/osu_benchmarks.py -n OSULatencyTest -L -p builtin -p gnu' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-7hnco47r.log' - -[List of matched checks] -- OSULatencyTest /14f35a43 [variant: 0, file: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py'] - ^OSUBuildTest /19b4fb56 [variant: 0, file: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py'] - ^OSUDownloadTest /7de668df [variant: 0, file: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py'] -Found 3 check(s) - -Log file(s) saved in '/tmp/rfm-7hnco47r.log' diff --git a/docs/listings/osu_latency_list.txt b/docs/listings/osu_latency_list.txt deleted file mode 100644 index 66840d6cc3..0000000000 --- a/docs/listings/osu_latency_list.txt +++ /dev/null @@ -1,18 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/deps/osu_benchmarks.py -n OSULatencyTest -l' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-7id7z75s.log' - -[List of matched checks] -- OSULatencyTest /14f35a43 - ^OSUBuildTest /19b4fb56 - ^OSUDownloadTest /7de668df -Found 3 check(s) - -Log file(s) saved in '/tmp/rfm-7id7z75s.log' diff --git a/docs/listings/osu_latency_unresolved_deps.txt b/docs/listings/osu_latency_unresolved_deps.txt deleted file mode 100644 index d82a5b97e0..0000000000 --- a/docs/listings/osu_latency_unresolved_deps.txt +++ /dev/null @@ -1,41 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/deps/osu_benchmarks.py -n OSULatencyTest --system=daint:gpu -l' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/deps/osu_benchmarks.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-12gjxnvc.log' - -WARNING: could not resolve dependency: ('OSUBuildTest', 'daint:gpu', 'gnu') -> 'OSUDownloadTest' -WARNING: could not resolve dependency: ('OSUBuildTest', 'daint:gpu', 'intel') -> 'OSUDownloadTest' -WARNING: could not resolve dependency: ('OSUBuildTest', 'daint:gpu', 'nvidia') -> 'OSUDownloadTest' -WARNING: skipping all dependent test cases - - ('OSUBuildTest', 'daint:gpu', 'intel') - - ('OSUBandwidthTest', 'daint:gpu', 'intel') - - ('OSUBuildTest', 'daint:gpu', 'nvidia') - - ('OSULatencyTest', 'daint:gpu', 'intel') - - ('OSUAllreduceTest_3', 'daint:gpu', 'nvidia') - - ('OSUBuildTest', 'daint:gpu', 'gnu') - - ('OSUAllreduceTest_1', 'daint:gpu', 'nvidia') - - ('OSUAllreduceTest_0', 'daint:gpu', 'intel') - - ('OSUAllreduceTest_2', 'daint:gpu', 'nvidia') - - ('OSUBandwidthTest', 'daint:gpu', 'gnu') - - ('OSULatencyTest', 'daint:gpu', 'gnu') - - ('OSUAllreduceTest_2', 'daint:gpu', 'intel') - - ('OSUAllreduceTest_3', 'daint:gpu', 'intel') - - ('OSUAllreduceTest_1', 'daint:gpu', 'intel') - - ('OSUAllreduceTest_0', 'daint:gpu', 'nvidia') - - ('OSUBandwidthTest', 'daint:gpu', 'nvidia') - - ('OSULatencyTest', 'daint:gpu', 'nvidia') - - ('OSUAllreduceTest_2', 'daint:gpu', 'gnu') - - ('OSUAllreduceTest_1', 'daint:gpu', 'gnu') - - ('OSUAllreduceTest_3', 'daint:gpu', 'gnu') - - ('OSUAllreduceTest_0', 'daint:gpu', 'gnu') - -[List of matched checks] -Found 0 check(s) - -Log file(s) saved in '/tmp/rfm-12gjxnvc.log' diff --git a/docs/listings/param_deps_list.txt b/docs/listings/param_deps_list.txt deleted file mode 100644 index 400efc053a..0000000000 --- a/docs/listings/param_deps_list.txt +++ /dev/null @@ -1,26 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/deps/parameterized.py -l' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/deps/parameterized.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-u9ryq5d3.log' - -[List of matched checks] -- TestB /cc291487 - ^TestA %z=9 /034f091a - ^TestA %z=8 /a093d19f - ^TestA %z=7 /77b4b8e6 - ^TestA %z=6 /40ce4759 -- TestA %z=5 /aa0cffc9 -- TestA %z=4 /83cd5dec -- TestA %z=3 /1c51609b -- TestA %z=2 /707b752c -- TestA %z=1 /c65657d5 -- TestA %z=0 /1b9f44df -Found 11 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-u9ryq5d3.log' diff --git a/docs/listings/perflogs.txt b/docs/listings/perflogs.txt deleted file mode 100644 index 5751f3a7a9..0000000000 --- a/docs/listings/perflogs.txt +++ /dev/null @@ -1,4 +0,0 @@ -job_completion_time,version,display_name,system,partition,environ,jobid,result,Copy_value,Copy_unit,Copy_ref,Copy_lower,Copy_upper,Scale_value,Scale_unit,Scale_ref,Scale_lower,Scale_upper,Add_value,Add_unit,Add_ref,Add_lower,Add_upper,Triad_value,Triad_unit,Triad_ref,Triad_lower,Triad_upper -2022-10-18T21:41:25,4.0.0-dev.2+90fbd3ef,StreamWithRefTest,catalina,default,gnu,81351,pass,24235.6,MB/s,25200,-0.05,0.05,16044.2,MB/s,16800,-0.05,0.05,17733.7,MB/s,18500,-0.05,0.05,18232.0,MB/s,18800,-0.05,0.05 -2022-10-18T21:41:31,4.0.0-dev.2+90fbd3ef,StreamWithRefTest,catalina,default,gnu,81377,fail,23615.4,MB/s,25200,-0.05,0.05,16394.5,MB/s,16800,-0.05,0.05,17841.3,MB/s,18500,-0.05,0.05,18284.1,MB/s,18800,-0.05,0.05 -2022-10-18T21:46:06,4.0.0-dev.2+90fbd3ef,StreamWithRefTest,catalina,default,gnu,81480,fail,23736.4,MB/s,25200,-0.05,0.05,16242.8,MB/s,16800,-0.05,0.05,17699.1,MB/s,18500,-0.05,0.05,18077.3,MB/s,18800,-0.05,0.05 diff --git a/docs/listings/run-report.json b/docs/listings/run-report.json deleted file mode 100644 index a31169ecd1..0000000000 --- a/docs/listings/run-report.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "session_info": { - "cmdline": "./bin/reframe -c tutorials/basics/hello/hello1.py -r", - "config_file": "", - "data_version": "2.0", - "hostname": "host", - "prefix_output": "/path/to/reframe/output", - "prefix_stage": "/path/to/reframe/stage", - "user": "user", - "version": "3.10.0-dev.3+c22440c1", - "workdir": "/path/to/reframe", - "time_start": "2022-01-22T13:21:50+0100", - "time_end": "2022-01-22T13:21:51+0100", - "time_elapsed": 0.8124568462371826, - "num_cases": 1, - "num_failures": 0 - }, - "runs": [ - { - "num_cases": 1, - "num_failures": 0, - "num_aborted": 0, - "num_skipped": 0, - "runid": 0, - "testcases": [ - { - "build_stderr": "rfm_HelloTest_build.err", - "build_stdout": "rfm_HelloTest_build.out", - "dependencies_actual": [], - "dependencies_conceptual": [], - "description": "HelloTest", - "display_name": "HelloTest", - "filename": "/path/to/reframe/tutorials/basics/hello/hello1.py", - "environment": "builtin", - "fail_phase": null, - "fail_reason": null, - "jobid": "43152", - "job_stderr": "rfm_HelloTest_job.err", - "job_stdout": "rfm_HelloTest_job.out", - "maintainers": [], - "name": "HelloTest", - "nodelist": [ - "tresa.local" - ], - "outputdir": "/path/to/reframe/output/generic/default/builtin/HelloTest", - "perfvars": null, - "prefix": "/path/to/reframe/tutorials/basics/hello", - "result": "success", - "stagedir": "/path/to/reframe/stage/generic/default/builtin/HelloTest", - "scheduler": "local", - "system": "generic:default", - "tags": [], - "time_compile": 0.27164483070373535, - "time_performance": 0.00010180473327636719, - "time_run": 0.3764667510986328, - "time_sanity": 0.0006909370422363281, - "time_setup": 0.007919073104858398, - "time_total": 0.8006880283355713, - "unique_name": "HelloTest" - } - ] - } - ], - "restored_cases": [] -} diff --git a/docs/listings/stream1.txt b/docs/listings/stream1.txt deleted file mode 100644 index 7cf37f4c2a..0000000000 --- a/docs/listings/stream1.txt +++ /dev/null @@ -1,40 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/basics/stream/stream1.py -r --performance-report' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/basics/stream/stream1.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-v0ig7jt4.log' - -[==========] Running 1 check(s) -[==========] Started on Sat Nov 12 19:00:53 2022 - -[----------] start processing checks -[ RUN ] StreamTest /cdf4820d @tresa:default+gnu -[ OK ] (1/1) StreamTest /cdf4820d @tresa:default+gnu -P: Copy: 24031.8 MB/s (r:0, l:None, u:None) -P: Scale: 16297.9 MB/s (r:0, l:None, u:None) -P: Add: 17843.8 MB/s (r:0, l:None, u:None) -P: Triad: 18278.3 MB/s (r:0, l:None, u:None) -[----------] all spawned checks have finished - -[ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Sat Nov 12 19:00:56 2022 - -================================================================================ -PERFORMANCE REPORT --------------------------------------------------------------------------------- -[StreamTest /cdf4820d @tresa:default:gnu] - num_tasks: 1 - num_gpus_per_node: 0 - performance: - - Copy: 24031.8 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 16297.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 17843.8 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 18278.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) --------------------------------------------------------------------------------- -Run report saved in '/home/user/.reframe/reports/run-report-324.json' -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-v0ig7jt4.log' diff --git a/docs/listings/stream3_failure_only.txt b/docs/listings/stream3_failure_only.txt deleted file mode 100644 index 77f6c2b219..0000000000 --- a/docs/listings/stream3_failure_only.txt +++ /dev/null @@ -1,14 +0,0 @@ -FAILURE INFO for StreamWithRefTest - * Expanded name: StreamWithRefTest - * Description: - * System partition: catalina:default - * Environment: gnu - * Stage directory: /Users/user/Repositories/reframe/stage/catalina/default/gnu/StreamWithRefTest - * Node list: tresa.localNone - * Job type: local (id=4576) - * Dependencies (conceptual): [] - * Dependencies (actual): [] - * Maintainers: [] - * Failing phase: performance - * Rerun with '-n /f925207b -p gnu --system catalina:default -r' - * Reason: performance error: failed to meet reference: Add=19585.3, expected 18500 (l=17575.0, u=19425.0) diff --git a/docs/listings/stream4_daint.txt b/docs/listings/stream4_daint.txt deleted file mode 100644 index e4796996f6..0000000000 --- a/docs/listings/stream4_daint.txt +++ /dev/null @@ -1,206 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2 - command: './bin/reframe -c tutorials/basics/stream/stream4.py -r --performance-report' - launched by: user@host - working directory: '/home/user/Devel/reframe' - settings files: '', '/home/user/Devel/reframe/tutorials/config/daint.py' - check search path: '/home/user/Devel/reframe/tutorials/basics/stream/stream4.py' - stage directory: '/home/user/Devel/reframe/stage' - output directory: '/home/user/Devel/reframe/output' - log files: '/tmp/rfm-yf6xjn_4.log' - -[==========] Running 1 check(s) -[==========] Started on Tue Nov 15 18:22:48 2022 - -[----------] start processing checks -[ RUN ] StreamMultiSysTest /eec1c676 @daint:login+gnu -[ RUN ] StreamMultiSysTest /eec1c676 @daint:login+intel -[ RUN ] StreamMultiSysTest /eec1c676 @daint:login+nvidia -[ RUN ] StreamMultiSysTest /eec1c676 @daint:login+cray -[ RUN ] StreamMultiSysTest /eec1c676 @daint:gpu+gnu -[ RUN ] StreamMultiSysTest /eec1c676 @daint:gpu+intel -[ RUN ] StreamMultiSysTest /eec1c676 @daint:gpu+nvidia -[ RUN ] StreamMultiSysTest /eec1c676 @daint:gpu+cray -[ RUN ] StreamMultiSysTest /eec1c676 @daint:mc+gnu -[ RUN ] StreamMultiSysTest /eec1c676 @daint:mc+intel -[ RUN ] StreamMultiSysTest /eec1c676 @daint:mc+nvidia -[ RUN ] StreamMultiSysTest /eec1c676 @daint:mc+cray -[ OK ] ( 1/12) StreamMultiSysTest /eec1c676 @daint:login+gnu -P: Copy: 97772.6 MB/s (r:0, l:None, u:None) -P: Scale: 69418.6 MB/s (r:0, l:None, u:None) -P: Add: 71941.0 MB/s (r:0, l:None, u:None) -P: Triad: 73679.7 MB/s (r:0, l:None, u:None) -[ OK ] ( 2/12) StreamMultiSysTest /eec1c676 @daint:login+intel -P: Copy: 85123.0 MB/s (r:0, l:None, u:None) -P: Scale: 79701.7 MB/s (r:0, l:None, u:None) -P: Add: 81632.7 MB/s (r:0, l:None, u:None) -P: Triad: 44391.5 MB/s (r:0, l:None, u:None) -[ OK ] ( 3/12) StreamMultiSysTest /eec1c676 @daint:login+nvidia -P: Copy: 76641.4 MB/s (r:0, l:None, u:None) -P: Scale: 59041.9 MB/s (r:0, l:None, u:None) -P: Add: 64792.5 MB/s (r:0, l:None, u:None) -P: Triad: 69441.4 MB/s (r:0, l:None, u:None) -[ OK ] ( 4/12) StreamMultiSysTest /eec1c676 @daint:login+cray -P: Copy: 35658.5 MB/s (r:0, l:None, u:None) -P: Scale: 27732.2 MB/s (r:0, l:None, u:None) -P: Add: 39037.7 MB/s (r:0, l:None, u:None) -P: Triad: 45310.3 MB/s (r:0, l:None, u:None) -[ OK ] ( 5/12) StreamMultiSysTest /eec1c676 @daint:gpu+gnu -P: Copy: 42666.3 MB/s (r:0, l:None, u:None) -P: Scale: 38491.0 MB/s (r:0, l:None, u:None) -P: Add: 43686.4 MB/s (r:0, l:None, u:None) -P: Triad: 43466.6 MB/s (r:0, l:None, u:None) -[ OK ] ( 6/12) StreamMultiSysTest /eec1c676 @daint:gpu+intel -P: Copy: 51726.7 MB/s (r:0, l:None, u:None) -P: Scale: 54185.6 MB/s (r:0, l:None, u:None) -P: Add: 57608.3 MB/s (r:0, l:None, u:None) -P: Triad: 57390.7 MB/s (r:0, l:None, u:None) -[ OK ] ( 7/12) StreamMultiSysTest /eec1c676 @daint:gpu+nvidia -P: Copy: 51810.8 MB/s (r:0, l:None, u:None) -P: Scale: 39653.4 MB/s (r:0, l:None, u:None) -P: Add: 44008.0 MB/s (r:0, l:None, u:None) -P: Triad: 44384.4 MB/s (r:0, l:None, u:None) -[ OK ] ( 8/12) StreamMultiSysTest /eec1c676 @daint:gpu+cray -P: Copy: 51101.8 MB/s (r:0, l:None, u:None) -P: Scale: 38568.1 MB/s (r:0, l:None, u:None) -P: Add: 43193.6 MB/s (r:0, l:None, u:None) -P: Triad: 43142.9 MB/s (r:0, l:None, u:None) -[ OK ] ( 9/12) StreamMultiSysTest /eec1c676 @daint:mc+gnu -P: Copy: 48292.9 MB/s (r:0, l:None, u:None) -P: Scale: 38499.5 MB/s (r:0, l:None, u:None) -P: Add: 43555.7 MB/s (r:0, l:None, u:None) -P: Triad: 43871.4 MB/s (r:0, l:None, u:None) -[ OK ] (10/12) StreamMultiSysTest /eec1c676 @daint:mc+cray -P: Copy: 46538.3 MB/s (r:0, l:None, u:None) -P: Scale: 40133.3 MB/s (r:0, l:None, u:None) -P: Add: 43363.9 MB/s (r:0, l:None, u:None) -P: Triad: 43450.3 MB/s (r:0, l:None, u:None) -[ OK ] (11/12) StreamMultiSysTest /eec1c676 @daint:mc+nvidia -P: Copy: 46648.2 MB/s (r:0, l:None, u:None) -P: Scale: 40384.5 MB/s (r:0, l:None, u:None) -P: Add: 44001.1 MB/s (r:0, l:None, u:None) -P: Triad: 44489.7 MB/s (r:0, l:None, u:None) -[ OK ] (12/12) StreamMultiSysTest /eec1c676 @daint:mc+intel -P: Copy: 51335.9 MB/s (r:0, l:None, u:None) -P: Scale: 49490.3 MB/s (r:0, l:None, u:None) -P: Add: 56859.9 MB/s (r:0, l:None, u:None) -P: Triad: 56544.5 MB/s (r:0, l:None, u:None) -[----------] all spawned checks have finished - -[ PASSED ] Ran 12/12 test case(s) from 1 check(s) (0 failure(s), 0 skipped) -[==========] Finished on Tue Nov 15 18:24:00 2022 - -================================================================================ -PERFORMANCE REPORT --------------------------------------------------------------------------------- -[StreamMultiSysTest /eec1c676 @daint:login:gnu] - num_cpus_per_task: 10 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 97772.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 69418.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 71941.0 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 73679.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:login:intel] - num_cpus_per_task: 10 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 85123.0 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 79701.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 81632.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 44391.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:login:nvidia] - num_cpus_per_task: 10 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 76641.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 59041.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 64792.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 69441.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:login:cray] - num_cpus_per_task: 10 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 35658.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 27732.2 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 39037.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 45310.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:gpu:gnu] - num_cpus_per_task: 12 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 42666.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 38491.0 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 43686.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 43466.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:gpu:intel] - num_cpus_per_task: 12 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 51726.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 54185.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 57608.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 57390.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:gpu:nvidia] - num_cpus_per_task: 12 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 51810.8 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 39653.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 44008.0 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 44384.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:gpu:cray] - num_cpus_per_task: 12 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 51101.8 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 38568.1 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 43193.6 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 43142.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:mc:gnu] - num_cpus_per_task: 36 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 48292.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 38499.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 43555.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 43871.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:mc:intel] - num_cpus_per_task: 36 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 51335.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 49490.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 56859.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 56544.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:mc:nvidia] - num_cpus_per_task: 36 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 46648.2 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 40384.5 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 44001.1 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 44489.7 MB/s (r: 0 MB/s l: -inf% u: +inf%) -[StreamMultiSysTest /eec1c676 @daint:mc:cray] - num_cpus_per_task: 36 - num_gpus_per_node: 0 - num_tasks: 1 - performance: - - Copy: 46538.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Scale: 40133.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Add: 43363.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - Triad: 43450.3 MB/s (r: 0 MB/s l: -inf% u: +inf%) --------------------------------------------------------------------------------- -Run report saved in '/home/user/.reframe/reports/run-report-2.json' -Log file(s) saved in '/tmp/rfm-yf6xjn_4.log' diff --git a/docs/listings/stream_params.txt b/docs/listings/stream_params.txt deleted file mode 100644 index 71836c914b..0000000000 --- a/docs/listings/stream_params.txt +++ /dev/null @@ -1,26 +0,0 @@ -[ReFrame Setup] - version: 4.0.0-dev.2+5ea6b7a6 - command: './bin/reframe -c tutorials/advanced/parameterized/stream.py -l' - launched by: user@host - working directory: '/home/user/Repositories/reframe' - settings files: '', '/home/user/Repositories/reframe/tutorials/config/tresa.py' - check search path: '/home/user/Repositories/reframe/tutorials/advanced/parameterized/stream.py' - stage directory: '/home/user/Repositories/reframe/stage' - output directory: '/home/user/Repositories/reframe/output' - log files: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-lou71c2g.log' - -[List of matched checks] -- StreamMultiSysTest %num_bytes=536870912 /cf10843f -- StreamMultiSysTest %num_bytes=268435456 /97fb363f -- StreamMultiSysTest %num_bytes=134217728 /7b4d01d3 -- StreamMultiSysTest %num_bytes=67108864 /530b0154 -- StreamMultiSysTest %num_bytes=33554432 /7199fc93 -- StreamMultiSysTest %num_bytes=16777216 /9d1b9ea8 -- StreamMultiSysTest %num_bytes=8388608 /3f29039f -- StreamMultiSysTest %num_bytes=4194304 /e30054cd -- StreamMultiSysTest %num_bytes=2097152 /45efaec5 -- StreamMultiSysTest %num_bytes=1048576 /92327981 -- StreamMultiSysTest %num_bytes=524288 /eb104cd0 -Found 11 check(s) - -Log file(s) saved in '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/rfm-lou71c2g.log' diff --git a/docs/listings/verbose_test_loading.txt b/docs/listings/verbose_test_loading.txt new file mode 100644 index 0000000000..3140584597 --- /dev/null +++ b/docs/listings/verbose_test_loading.txt @@ -0,0 +1,108 @@ +Loading user configuration +Loading the builtin configuration +Loading configuration file: 'config/baseline_environs.py' +Autodetecting system +Trying autodetection method: 'py::socket.gethostname' +Retrieved hostname: 'myhost' +Looking for a matching configuration entry +Configuration found: picking system 'tutorialsys' +Initializing runtime +Initializing system partition 'default' +Initializing system 'tutorialsys' +Initializing modules system 'nomod' +detecting topology info for tutorialsys:default +> found topology file '/home/user/.reframe/topology/tutorialsys-default/processor.json'; loading... +> device auto-detection is not supported +[ReFrame Environment] + RFM_AUTODETECT_FQDN= + RFM_AUTODETECT_METHOD= + RFM_AUTODETECT_METHODS= + RFM_AUTODETECT_XTHOSTNAME= + RFM_CHECK_SEARCH_PATH= + RFM_CHECK_SEARCH_RECURSIVE= + RFM_CLEAN_STAGEDIR= + RFM_COLORIZE= + RFM_COMPRESS_REPORT= + RFM_CONFIG_FILES= + RFM_CONFIG_PATH= + RFM_DUMP_PIPELINE_PROGRESS= + RFM_GIT_TIMEOUT= + RFM_HTTPJSON_URL= + RFM_IGNORE_REQNODENOTAVAIL= + RFM_INSTALL_PREFIX=/usr/local/share/reframe + RFM_KEEP_STAGE_FILES= + RFM_MODULE_MAPPINGS= + RFM_MODULE_MAP_FILE= + RFM_NON_DEFAULT_CRAYPE= + RFM_OUTPUT_DIR= + RFM_PERFLOG_DIR= + RFM_PERF_INFO_LEVEL= + RFM_PIPELINE_TIMEOUT= + RFM_PREFIX= + RFM_PURGE_ENVIRONMENT= + RFM_REMOTE_DETECT= + RFM_REMOTE_WORKDIR= + RFM_REPORT_FILE= + RFM_REPORT_JUNIT= + RFM_RESOLVE_MODULE_CONFLICTS= + RFM_SAVE_LOG_FILES= + RFM_STAGE_DIR= + RFM_SYSLOG_ADDRESS= + RFM_SYSTEM= + RFM_TIMESTAMP_DIRS= + RFM_TRAP_JOB_ERRORS= + RFM_UNLOAD_MODULES= + RFM_USER_MODULES= + RFM_USE_LOGIN_SHELL= + RFM_VERBOSE= +[ReFrame Setup] + version: 4.6.0-dev.2 + command: '/usr/local/share/reframe/bin/reframe -C config/baseline_environs.py -c stream/stream_variables.py -l -vv' + launched by: user@myhost + working directory: '/home/user/reframe-examples/tutorial' + settings files: '', 'config/baseline_environs.py' + check search path: '/home/user/reframe-examples/tutorial/stream/stream_variables.py' + stage directory: '/home/user/reframe-examples/tutorial/stage' + output directory: '/home/user/reframe-examples/tutorial/output' + log files: '/tmp/rfm-f2v37wl4.log' + +Looking for tests in '/home/user/reframe-examples/tutorial/stream/stream_variables.py' +Validating '/home/user/reframe-examples/tutorial/stream/stream_variables.py': OK + > Loaded 3 test(s) +Loaded 3 test(s) +Generated 4 test case(s) +Filtering test cases(s) by name: 2 remaining +Filtering test cases(s) by tags: 2 remaining +Filtering test cases(s) by other attributes: 2 remaining +Building and validating the full test DAG +Full test DAG: + ('stream_test', 'tutorialsys:default', 'gnu') -> [('build_stream_32608d67', 'tutorialsys:default', 'gnu')] + ('stream_test', 'tutorialsys:default', 'clang') -> [('build_stream_de1600df', 'tutorialsys:default', 'clang')] + ('build_stream_de1600df', 'tutorialsys:default', 'clang') -> [] + ('build_stream_32608d67', 'tutorialsys:default', 'gnu') -> [] +Pruned test DAG + ('stream_test', 'tutorialsys:default', 'gnu') -> [('build_stream_32608d67', 'tutorialsys:default', 'gnu')] + ('build_stream_32608d67', 'tutorialsys:default', 'gnu') -> [] + ('stream_test', 'tutorialsys:default', 'clang') -> [('build_stream_de1600df', 'tutorialsys:default', 'clang')] + ('build_stream_de1600df', 'tutorialsys:default', 'clang') -> [] +Final number of test cases: 4 +[List of matched checks] +- stream_test /2e15a047 + ^build_stream ~tutorialsys:default+gnu 'stream_binary /40af02af + ^build_stream ~tutorialsys:default+clang 'stream_binary /8effd276 +Found 1 check(s) + +Log file(s) saved in '/tmp/rfm-f2v37wl4.log' +>>> profiler report [start] <<< +main: 0.118834 s + test processing: 0.037243 s + RegressionCheckLoader.load_all: 0.028813 s + TestRegistry.instantiate_all: 0.015122 s + generate_testcases: 0.000090 s + main.._sort_testcases: 0.000019 s + build_deps: 0.000259 s + validate_deps: 0.000097 s + prune_deps: 0.000199 s + toposort: 0.000200 s + list_checks: 0.002932 s +>>> profiler report [ end ] <<< diff --git a/docs/manpage.rst b/docs/manpage.rst index 2b56ef1080..387c27f9f0 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -12,7 +12,7 @@ Synopsis Description ----------- -ReFrame provides both a `programming interface `__ for writing regression tests and a command-line interface for managing and running the tests, which is detailed here. +ReFrame provides both a :doc:`programming interface ` for writing regression tests and a command-line interface for managing and running the tests, which is detailed here. The ``reframe`` command is part of ReFrame's frontend. This frontend is responsible for loading and running regression tests written in ReFrame. ReFrame executes tests by sending them down to a well defined pipeline. @@ -251,7 +251,7 @@ An action must always be specified. For more information, have a look in :ref:`generate-ci-pipeline`. .. note:: - This option will not work with the `test generation options <#test-generators>`. + This option will not work with the :ref:`test generation options `. .. versionadded:: 3.4.1 @@ -385,7 +385,7 @@ Options controlling ReFrame output Directory prefix for logging performance data. - This option is relevant only to the ``filelog`` `logging handler `__. + This option is relevant only to the ``filelog`` :ref:`logging handler `. This option can also be set using the :envvar:`RFM_PERFLOG_DIR` environment variable or the :attr:`~config.logging.handlers_perflog..filelog..basedir` logging handler configuration parameter. @@ -441,7 +441,7 @@ Options controlling ReFrame output Save ReFrame log files in the output directory before exiting. - Only log files generated by ``file`` `log handlers `__ will be copied. + Only log files generated by ``file`` :ref:`log handlers ` will be copied. This option can also be set using the :envvar:`RFM_SAVE_LOG_FILES` environment variable or the :attr:`~config.general.save_log_files` general configuration parameter. @@ -583,7 +583,7 @@ Options controlling ReFrame execution This is not a problem when rerunning a failed case, since the stage directories of its dependencies are automatically kept, but if you want to rerun a successful test case, you should make sure to have run with the :option:`--keep-stage-files` option. .. note:: - This option will not work with the `test generation options <#test-generators>`. + This option will not work with the :ref:`test generation options `. .. versionadded:: 3.4 @@ -752,6 +752,18 @@ If no node can be selected, the test will be marked as a failure with an appropr Slurm OR constraints and parenthesized expressions are supported in flexible node allocation. + .. versionchanged:: 4.7 + The test is not marked as a failure if not enough nodes are available, but it is skipped instead. + To enforce a failure, use :option:`--flex-alloc-strict` + +.. option:: --flex-alloc-strict + + Fail flexible tests if their minimum task requirement is not satisfied. + Otherwise the tests will be skipped. + + .. versionadded:: 4.7 + + --------------------------------------- Options controlling ReFrame environment --------------------------------------- @@ -1175,7 +1187,7 @@ Notice that this example leads to a name conflict with the old naming scheme, si Each test is also associated with a hash code that is derived from the test name, its parameters and their values. As in the example listing above, the hash code of each test is printed with the :option:`-l` option and individual tests can be selected by their hash using the :option:`-n` option, e.g., ``-n /1c51609b``. -The stage and output directories, as well as the performance log file of the ``filelog`` `performance log handler `__ will use the hash code for the test-specific directories and files. +The stage and output directories, as well as the performance log file of the ``filelog`` :ref:`performance log handler ` will use the hash code for the test-specific directories and files. This might lead to conflicts for tests as the one above when executing them with the asynchronous execution policy, but ensures consistency of performance record files when parameter values are added to or deleted from a test parameter. More specifically, the test's hash will not change if a new parameter value is added or deleted or even if the parameter values are shuffled. Test variants on the other side are more volatile and can change with such changes. @@ -1425,6 +1437,21 @@ Whenever an environment variable is associated with a configuration option, its .. versionadded:: 4.0.0 +.. envvar:: RFM_FLEX_ALLOC_STRICT + + Fail flexible tests if their minimum task requirement is not satisfied. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--flex-alloc-strict` + Associated configuration parameter :attr:`~config.general.flex_alloc_strict` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_GIT_TIMEOUT Timeout value in seconds used when checking if a git repository exists. @@ -1858,13 +1885,10 @@ If no configuration file can be found in any of the predefined locations, ReFram This configuration file is located in |reframe/core/settings.py|_. Users may *not* modify this file. -For a complete reference of the configuration, please refer to |reframe.settings(8)|_ man page. +For a complete reference of the configuration, please refer to :doc:`reframe.settings(8) ` man page. .. |reframe/core/settings.py| replace:: ``reframe/core/settings.py`` .. _reframe/core/settings.py: https://github.com/reframe-hpc/reframe/blob/master/reframe/core/settings.py -.. |reframe.settings(8)| replace:: ``reframe.settings(8)`` -.. _reframe.settings(8): config_reference.html - Reporting Bugs -------------- diff --git a/docs/pipeline.rst b/docs/pipeline.rst index b6efb3edef..1bea1b4b95 100644 --- a/docs/pipeline.rst +++ b/docs/pipeline.rst @@ -14,7 +14,7 @@ The following figure explains in more detail the process: When ReFrame loads a test from the disk it unconditionally constructs it executing its :func:`__init__` method. The practical implication of this is that your test will be instantiated even if it will not run on the current system. -After all the tests are loaded, they are filtered based on the current system and any other criteria (such as programming environment, test attributes etc.) specified by the user (see `Test Filtering `__ for more details). +After all the tests are loaded, they are filtered based on the current system and any other criteria (such as programming environment, test attributes etc.) specified by the user (see :ref:`Test Filtering ` for more details). After the tests are filtered, ReFrame creates the actual `test cases` to be run. A test case is essentially a tuple consisting of the test, the system partition and the programming environment to try. The test that goes into a test case is essentially a `clone` of the original test that was instantiated upon loading. This ensures that the test case's state is not shared and may not be reused in any case. @@ -66,7 +66,7 @@ The Run Phase During this phase a job script associated with the test case will be created and it will be submitted for execution. If the test is `"run-only," `__ its `resources `__ will be first copied to the test case's stage directory. ReFrame will temporarily switch to that directory and spawn the test's job from there. -This phase is executed asynchronously (either a batch job is spawned or a local process is started) and it is up to the selected `execution policy <#execution-policies>`__ to block or not until the associated job finishes. +This phase is executed asynchronously (either a batch job is spawned or a local process is started) and it is up to the selected :ref:`execution policy ` to block or not until the associated job finishes. ---------------- @@ -96,6 +96,8 @@ More specifically, if the test has finished successfully, all interesting test f This phase might be deferred in case a test has dependents (see :ref:`cleaning-up-stage-files` for more details). +.. _execution-policies: + Execution Policies ------------------ diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index 5b546b3e36..16c38feab1 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -289,7 +289,7 @@ Modules Systems :members: :show-inheritance: - +.. _build-systems: ------------- Build Systems ------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index 0a7408c2c2..f589cdd2fc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,9 +1,9 @@ -archspec==0.2.2 +archspec==0.2.4 docutils==0.18.1 jsonschema==3.2.0 semver==2.13.0; python_version == '3.6' semver==3.0.2; python_version >= '3.7' Sphinx==5.3.0; python_version < '3.8' Sphinx==7.1.2; python_version == '3.8' -Sphinx==7.2.6; python_version >= '3.9' +Sphinx==7.3.7; python_version >= '3.9' sphinx-rtd-theme==2.0.0 diff --git a/docs/started.rst b/docs/started.rst index e3f9c0507e..ccfc1f57ba 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -105,6 +105,10 @@ The ``./bootstrap.sh`` has two additional variant options: The bootstrap script for ReFrame was added. For previous ReFrame versions you should install its requirements using ``pip install -r requirements.txt`` in a Python virtual environment. + .. versionchanged:: 4.5 + ReFrame now supports multiarch builds and it will place all of its dependencies in an arch-specific directory under its prefix. + Also, ``pip`` is no more required, as the bootstrap script will start a virtual environment without ``pip`` and will fetch a fresh ``pip``, which will be used to install the dependencies. + Enabling auto-completion ------------------------ @@ -123,10 +127,8 @@ Auto-completion is supported for Bash, Tcsh and Fish shells. Where to Go from Here --------------------- -If you are new to ReFrame, the place to start is the first tutorial :doc:`tutorial_basics`, which will guide you step-by-step in both writing your first tests and in configuring ReFrame. -The rest of the tutorials explore additional capabilities of the framework and cover several topics that you will likely come across when writing your own tests. - -The :doc:`configure` page provides more details on how a configuration file is structured and the :doc:`topics` explain some more advanced concepts as well as some implementation details. -The :doc:`manuals` provide complete reference guides for the command line interface, the configuration parameters and the programming APIs for writing tests. +If you are new to ReFrame, the place to start is the :doc:`tutorial`, which will guide you through all the concepts of the framework and get you up and running. +If you are looking for a particular topic that is not covered in the tutorial, you can refer to the :doc:`howto` or the :doc:`topics`. +For detailed reference guides for the command line, the configuration and the programming API, refer to the :doc:`manuals`. -Finally, if you are not new to ReFrame and you have been using the 3.x versions, you should read the :doc:`whats_new_40` page, which explains what are the key new features of ReFrame 4.0 as well as all the breaking changes. +Finally, if you are already a user of ReFrame 3.x version, you should read the :doc:`whats_new_40` page, which explains what are the key new features of ReFrame 4.0 as well as all the breaking changes. diff --git a/docs/tutorial.rst b/docs/tutorial.rst new file mode 100644 index 0000000000..9852171216 --- /dev/null +++ b/docs/tutorial.rst @@ -0,0 +1,1955 @@ +.. currentmodule:: reframe.core.pipeline.RegressionTest + +================ +ReFrame Tutorial +================ + +This tutorial will cover the basic concepts of ReFrame and will get you started with the framework. +For more specific topics, you should refer to ":doc:`howto`" as well as to the ":doc:`topics`" for an in-depth understanding of some of the framework's concepts. + +.. contents:: Table of Contents + :local: + :depth: 3 + +Requirements +============ + +To run this tutorial you need ``docker`` for the local examples and ``docker compose`` for the examples emulating a Slurm cluster. +Note, that the Docker daemon must be running. + +The tutorial container images provide already the latest ReFrame version installed. +For installing a stand-alone version of ReFrame, please refer to the ":doc:`started`" guide. + +All tutorial examples are located under the ``reframe-examples`` directory inside the container's working directory. + + +Running the local examples +-------------------------- + +To run the local examples, launch the single-node tutorial container by binding mounting the examples: + +.. code-block:: bash + + git clone https://github.com/reframe-hpc/reframe.git + cd reframe + docker build -t reframe-tut-singlenode:latest -f examples/tutorial/dockerfiles/singlenode.Dockerfile . + docker run -h myhost -it --mount type=bind,source=$(pwd)/examples/,target=/home/user/reframe-examples reframe-tut-singlenode:latest /bin/bash + + +.. _multi-node-setup: + +Running the multi-node examples +------------------------------- + +To run the multi-node examples you need first to launch a Slurm pseudo cluster using the provided Docker compose file: + +.. code-block:: bash + + git clone https://github.com/reframe-hpc/reframe.git + cd reframe + docker compose --project-directory=$(pwd) -f examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend + +Once the Docker compose stack is up, you execute the following from a different terminal window in order to "log in" in the frontend container: + +.. code-block:: + + docker exec -it $(docker ps -f name=frontend -q) /bin/bash + + # Inside the container + cd reframe-examples/tutorial/ + +Once done, press Ctl-D in the frontend container and Ctl-C in the Docker compose console window. + + +.. note:: + + All examples use the single-node container unless it is otherwise noted. + + +Modifying the examples +---------------------- + +In both cases, the tutorial examples are bind mounted in the container, so you could make changes directly from your host and these will be reflected inside the container and vice versa. + + +.. _writing-your-first-test: + +Writing your first test +======================= + +We will start with the `STREAM benchmark `__. +This is a standard benchmark for measuring the DRAM bandwidth. +The tutorial container already contains a pre-compiled OpenMP version of the benchmark. +Our test will run the STREAM executable, validate the output and extract the figure of merits. +Here is the full ReFrame test: + +.. literalinclude:: ../examples/tutorial/stream/stream_runonly.py + :caption: + :lines: 5- + +ReFrame tests are specially decorated classes that ultimately derive from the :class:`~reframe.core.pipeline.RegressionTest` class. +Since we only want to run an executable in this first test, we derive from the :class:`~reframe.core.pipeline.RunOnlyRegressionTest` class, which essentially short-circuits the "compile" stage of the test. +The :func:`@simple_test ` decorator registers a test class with the framework and makes it available for running. + +Every ReFrame test must define the :attr:`valid_systems` and :attr:`valid_prog_environs` variables. +These describe the test's constraints and the framework will automatically filter out tests on systems and environments that do not match the constraints. +We will describe the system and environments abstractions later in this tutorial. +For this first example, the ``*`` symbol denotes that this test is valid for any system or environment. +A :class:`~reframe.core.pipeline.RunOnlyRegressionTest` must also define an :attr:`executable` to run. + +A test must also define a validation function which is decorated with the :func:`@sanity_function` decorator. +This function will be used to validate the test's output after it is finished. +ReFrame, by default, makes no assumption about whether a test is successful or not; +it is the test's responsibility to define its validation. +The framework provides a rich set of :doc:`utility functions ` that help matching patterns and extract values from the test's output. +The :attr:`stdout` here refers to the name of the file where the test's standard output is stored. + +Finally, a test may optionally define a set of *performance functions* that will extract *figures of merit* for the test. +These are simple test methods decorated with the :func:`@performance_function ` decorator that return the figure of merit. +In this example, we extract the ``Copy`` and ``Triad`` bandwidth values and convert them to ``float``. +These figures of merit or *performance variables* as they are called in ReFrame's nomenclature have a special treatment: +they are logged in the test's performance logs and a reference value per system may also be assigned to them. +If that reference value is not met within some user-defined thresholds, the test will fail. + + +Running a test +============== + +Running our test is very straightforward: + +.. code-block:: bash + :caption: Run in the single-node container. + + cd reframe-examples/tutorial + reframe -c stream/stream_runonly.py -r + + +The :option:`-c` option defines the *check path* and it can be specified multiple times. +It specifies the locations, directories or files, where ReFrame will try to look for tests. +In this case, we simply pass the path to our test file. +The :option:`-r` option instructs ReFrame to run the selected tests: + +.. code-block:: console + + [ReFrame Setup] + version: 4.5.0-dev.1 + command: '/usr/local/share/reframe/bin/reframe -c stream/stream_runonly.py -r' + launched by: user@myhost + working directory: '/home/user' + settings files: '' + check search path: '/home/user/reframe-examples/tutorial/stream/stream_runonly.py' + stage directory: '/home/user/stage' + output directory: '/home/user/output' + log files: '/tmp/rfm-mzynqhye.log' + + [==========] Running 1 check(s) + [==========] Started on Mon Nov 27 20:55:17 2023 + + [----------] start processing checks + [ RUN ] stream_test /2e15a047 @generic:default+builtin + [ OK ] (1/1) stream_test /2e15a047 @generic:default+builtin + P: copy_bw: 19538.4 MB/s (r:0, l:None, u:None) + P: triad_bw: 14883.4 MB/s (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped, 0 aborted) + [==========] Finished on Mon Nov 27 20:55:25 2023 + Log file(s) saved in '/tmp/rfm-mzynqhye.log' + + +The verbosity of the output can be increasing using the :option:`-v` option or decreased using the :option:`-q` option. +By default, a log file is generated in the system's temporary directory that contains detailed debug information. +The :ref:`logging` section article describes how logging can be configured in more detail. +Once a performance test finishes, its figures of merit are printed immediately using the ``P:`` prefix. +This can be suppressed by increasing the level at which this information is logged using the :envvar:`RFM_PERF_INFO_LEVEL` environment variable. + +.. _run-reports-and-performance-logging: + +Run reports and performance logging +----------------------------------- + +Once a test session finishes, ReFrame generates a detailed JSON report under ``$HOME/.reframe/reports``. +Every time ReFrame is run a new report will be generated automatically. +The latest one is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given. + +For performance tests, in particular, an additional CSV file is generated with all the relevant information. +These files are located by default under ``perflogs///.log``. +In our example, this translates to ``perflogs/generic/default/stream_test.log``. +The information that is being logged is fully configurable and we will cover this in the :ref:`logging` section. + +Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session. + +.. code-block:: console + + [stream_test /2e15a047 @generic:default:builtin] + num_tasks: 1 + performance: + - copy_bw: 22704.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) + - triad_bw: 16040.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) + + +Inspecting the test artifacts +----------------------------- + +When ReFrame executes tests, it first copies over all of the test resources (if any) to a *stage directory*, from which it executes the test. +Upon successful execution, the test artifacts will be copied over to the *output directory* for archiving. +The default artifacts for every test are the generated test script as well as the test's standard output and standard error. +The default location for the stage and output directories are the ``./stage`` and ``./output`` directories. +These can be changed with the :option:`--stage` and :option:`--output` options or the more general :option:`--prefix` option. +The test artifacts of our first example can be found in the following location: + +.. code-block:: bash + :caption: Run in the single-node container. + + ls output/generic/default/builtin/stream_test/ + +.. code-block:: console + + rfm_job.err rfm_job.out rfm_job.sh + +The ``rfm_job.sh`` is the actual test script that was generated and executed and, as you can see, it was pretty simple for this case: + +.. code-block:: bash + + #!/bin/bash + stream.x + +Inspecting test failures +------------------------- + +When a test fails, ReFrame will not move its artifacts to the output directory and will keep everything inside the stage directory. +For each failed test, a summary will be printed at the end that contains details about the reason of the failure and the location of the test's stage directory. +Here is an example failure that we induced artificially by changing the validation regular expression: + +.. code-block:: console + + FAILURE INFO for stream_test (run: 1/1) + * Description: + * System partition: generic:default + * Environment: builtin + * Stage directory: /home/user/stage/generic/default/builtin/stream_test + * Node list: myhost + * Job type: local (id=19) + * Dependencies (conceptual): [] + * Dependencies (actual): [] + * Maintainers: [] + * Failing phase: sanity + * Rerun with '-n /2e15a047 -p builtin --system generic:default -r' + * Reason: sanity error: pattern 'Slution Validates' not found in 'rfm_job.out' + --- rfm_job.out (first 10 lines) --- + ------------------------------------------------------------- + STREAM version $Revision: 5.10 $ + ------------------------------------------------------------- + This system uses 8 bytes per array element. + ------------------------------------------------------------- + Array size = 100000000 (elements), Offset = 0 (elements) + Memory per array = 762.9 MiB (= 0.7 GiB). + Total memory required = 2288.8 MiB (= 2.2 GiB). + Each kernel will be executed 10 times. + The *best* time for each kernel (excluding the first iteration) + --- rfm_job.out --- + --- rfm_job.err (first 10 lines) --- + --- rfm_job.err --- + + +Adding performance references +----------------------------- + +For each performance variable defined in the test, we can add a reference value and set thresholds of acceptable variations. +Here is an example for our STREAM benchmark: + +.. code-block:: python + + @rfm.simple_test + class stream_test(rfm.RunOnlyRegressionTest): + ... + reference = { + 'myhost:baseline': { + 'copy_bw': (23_890, -0.10, 0.30, 'MB/s'), + 'triad_bw': (17_064, -0.05, 0.50, 'MB/s'), + } + } + +The :attr:`reference` test variable is a multi-level dictionary that defines the expected performance for each of the test's performance variables on all supported systems. +It is not necessary that all performance variables and all systems have a reference. +If a reference value is not found, then the obtained performance will be logged, but no performance validation will be performed. +The reference value is essentially a three or four element tuple of the form: ``(target_perf, lower_thres, upper_thres, unit)``. The ``unit`` is optional as it is already defined in the :func:`@performance_function ` definitions. +The lower and upper thresholds are deviations from the target reference expressed as fractional numbers. +In our example, we allow the ``copy_bw`` to be 10% lower than the target reference and no more than 30% higher. +Sometimes, especially in microbenchmarks, it is a good practice to set an upper threshold to denote the absolute maximum that cannot be exceeded. + + +Dry-run mode +------------ + +ReFrame provides also a dry-run mode for the tests, which can be enabled by passing :option:`--dry-run` as the action option (instead of :option:`-r` that runs the tests). +In this mode, ReFrame will generate the test script to be executed in the stage directory, but it will not run the test and will not perform the sanity and performance checking, neither will it attempt to extract any of the figures of merit. +Tests can also modify their behaviour if run in dry-run mode by calling the :meth:`is_dry_run` method. +Here is an example dry-run of our first version of the STREAM benchmark: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -c stream/stream_runonly.py --dry-run + +.. code-block:: console + + [==========] Running 1 check(s) + [==========] Started on Wed Jan 10 22:45:49 2024+0000 + + [----------] start processing checks + [ DRY ] stream_test /2e15a047 @generic:default+builtin + [ OK ] (1/1) stream_test /2e15a047 @generic:default+builtin + [----------] all spawned checks have finished + + [ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped, 0 aborted) + [==========] Finished on Wed Jan 10 22:45:49 2024+0000 + + +Note that the ``RUN`` message is replaced by ``DRY`` in the dry-run mode. +You can also check the generated test script in ``stage/generic/default/builtin/stream_test/rfm_job.sh``. + + +Systems and environments +======================== + +The first version of our STREAM test assumes that the environment it runs in already provides the benchmark executable. +This is totally fine if we want to run a set of run-only tests in a single environment, but it can become a maintenance burden if we want to run our test on different systems and environments. +In this section, we will introduce how we can define system and environment configurations in ReFrame and match them to tests. + +For ReFrame, a *system* is an abstraction of an HPC system that is managed by a workload manager. +A system can comprise multiple *partitions*, which are collection of nodes with similar characteristics. +This is entirely up to the user on how to define the system partitions. + +An *environment* is an abstraction of the environment where a test will run and it is a collection of environment variables, environment modules and compiler definitions. +The following picture depicts this architecture. + +.. figure:: _static/img/reframe-system-arch.svg + :align: center + + :sub:`ReFrame's system architecture` + +Tests are associated with systems and environments through their :attr:`valid_systems` and :attr:`valid_prog_environs` variables. + +Let's limit the scope of our test by making it require a specific environment, since in order to run it we require an environment that provides STREAM. +We could do that simply by setting the :attr:`valid_prog_environs` as follows: + +.. code-block:: python + + self.valid_prog_environs = ['+stream'] + +This tells ReFrame that this test is valid only for environments that define the ``stream`` feature. +If we try to run the test now, nothing will be run: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -c stream/stream_runonly.py -r + + +.. code-block:: console + + [ PASSED ] Ran 0/0 test case(s) from 0 check(s) (0 failure(s), 0 skipped, 0 aborted) + + +This happens because ReFrame by default defines a generic system and environment. +You may have noticed in our first run the ``@generic:default+builtin`` notation printed after test name. +This is the system partition name (``generic:default``) and the environment name (``builtin``) where the test is being run in. +The ``generic`` system and the ``builtin`` partition come as predefined in ReFrame. +They make the minimum possible assumptions: + +- The ``generic`` system defines a single partition, named ``default`` which launches test jobs locally. +- The ``builtin`` environment assumes only that the ``cc`` compiler is available. + +.. note:: + ReFrame will not complain if a compiler is not installed until your test tries to build something. + + +Let's define our own system and baseline environment in a ReFrame configuration file (``reframe-examples/tutorial/config/baseline.py``): + +.. literalinclude:: ../examples/tutorial/config/baseline.py + :caption: + :lines: 5- + +This configuration defines a system named ``tutorialsys`` with a single partition named ``default`` and an environment named ``baseline``. +Let's look at some key elements of the configuration: + +* Each system, partition and environment require a unique name. + The name must contain only alphanumeric characters, underscores or dashes. +* The :attr:`~config.systems.hostnames` option defines a set of hostname patterns which ReFrame will try to match against the current system's hostname. + The first matching system will become the current system and ReFrame will load the corresponding configuration. +* The :attr:`~config.systems.partitions.scheduler` partition option defines the job scheduler backend to use on this partition. + ReFrame supports many `job schedulers `__. + The ``local`` scheduler that we use here is the simplest one and it practically spawns a process executing the generated test script. +* The :attr:`~config.systems.partitions.launcher` partition option defines the parallel launcher to use for spawning parallel programs. + ReFrame supports all the major `parallel launchers `__. +* The :attr:`~config.systems.partitions.environs` partition option is a list of environments to test on this partition. + Their definitions are resolved in the :attr:`~config.environments` section. +* Every partition and environment can define a set of arbitrary features or key/value pairs in the :attr:`~config.environments.features` and :attr:`~config.environments.extras` options respectively. + ReFrame will try to match system partitions and environments to a test based on the test's specification in :attr:`valid_systems` and :attr:`valid_prog_environs`. + +There are many options that we can be define for systems, partitions and environments. +We will cover several of them as we go through the tutorial, but for the complete reference you should refer to :doc:`config_reference`. + +.. note:: + + ReFrame supports splitting the configuration in multiple files that can be loaded simultaneously. + In fact the builtin configuration is always loaded, therefore the ``generic`` system as well as the ``builtin`` environment are always defined. + Additionally, the builtin configuration provides a baseline logging configuration that should cover a wide range of use cases. + See :ref:`managing-the-configuration` for more details. + + +Let's try running the constrained version of our STREAM test with the configuration file that we have just created: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline.py -c stream/stream_runonly.py -r + +.. code-block:: console + + [ReFrame Setup] + version: 4.5.0-dev.1 + command: '/usr/local/share/reframe/bin/reframe -C config/baseline.py -c stream/stream_runonly.py -r' + launched by: user@myhost + working directory: '/home/user' + settings files: '', 'reframe-examples/tutorial/config/baseline.py' + check search path: '/home/user/reframe-examples/tutorial/stream/stream_runonly.py' + stage directory: '/home/user/stage' + output directory: '/home/user/output' + log files: '/tmp/rfm-dz8m5nfz.log' + + <...> + + [----------] start processing checks + [ RUN ] stream_test /2e15a047 @tutorialsys:default+baseline + [ OK ] (1/1) stream_test /2e15a047 @tutorialsys:default+baseline + P: copy_bw: 23135.4 MB/s (r:0, l:None, u:None) + P: triad_bw: 16600.5 MB/s (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped, 0 aborted) + + +The :option:`-C` option specifies the configuration file that ReFrame should load. +Note that ReFrame has loaded two configuration files: first the ```` and then the one we supplied. + +Note also that the system and environment specification in the test run output is now ``@tutorialsys:default+baseline``. +ReFrame has determined that the ``default`` partition and the ``baseline`` environment satisfy the test constraints and thus it has run the test with this partition/environment combination. + + +.. _compiling-the-test-code: + +Compiling the test code +======================= + +You can also use ReFrame to compile the test's code. +To demonstrate this, we will write a different test version of the STREAM benchmark that will also compile the benchmark's source code. + +.. literalinclude:: ../examples/tutorial/stream/stream_build_run.py + :caption: + :pyobject: stream_build_test + +The key difference of this test is that it derives from the :class:`~reframe.core.pipeline.RegressionTest` instead of the :class:`~reframe.core.pipeline.RunOnlyRegressionTest` and it specifies how the test's code should be built. +ReFrame uses a *build system* abstraction for building source code. +Based on the build system backend used, it will emit the appropriate build instructions. +All the major build systems are supported as well as the `EasyBuild `__ build automation tool and the `Spack `__ package manager. + +In this case, we use the :class:`~reframe.core.build_systems.SingleSource` build system, which is suitable for compiling a single source file. +The :attr:`sourcepath` variable is used to specify the source file to compile. +The path is relative to the test's *resource directory*. + +Test resources +-------------- + +The resource directory is a directory associated to the test where its static resources are stored. +During execution the contents of this directory will be copied to the test's stage directory and the test will execute from that directory. +Here is the directory structure: + +.. code:: + + stream + ├── stream_build_run.py + └── src + └── stream.c + +By default, the test's resources directory is named ``src/`` and is located next to the test's file. +It can be set to a different location inside the test using the :attr:`sourcesdir` variable. + +Pipeline hooks +-------------- + +The :func:`prepare_build` test method in our example is a *pipeline hook* that will execute just before the compilation phase and will set the compilation flags based on the current environment. +Pipeline hooks are a fundamental tool in ReFrame for customizing the test execution. +Let's explain the concept in more detail. + +When executed, every test in ReFrame goes through the following stages: +(a) *setup*, +(b) *compile*, +(c) *run*, +(d) *sanity*, +(e) *performance* and +(f) *cleanup*. +This is the *test pipeline* and a test can assign arbitrary functions to run before or after any of these stages using the :func:`@run_before ` and :func:`@run_after ` decorators. +There is also a pseudo-stage called *init* that denotes the instantiation/initialization of the test. +The :doc:`pipeline` page describes in detail every stage, but the most important stages in terms of the test's lifetime are the "init" and the "setup" stages. + +The "init" stage is where the test object is actually instantiated and for this reason you cannot define a pre-init hooks. +At this stage, the system partition and the environment where the test will run are not yet determined, therefore the :attr:`current_partition` and :attr:`current_environ` variables are not set. +This happens during the "setup" stage, where also all the test's dependencies (if any) have been executed and their resources can be safely accessed (we will cover test dependencies later in this tutorial). +Technically, all pipeline hooks could be attached to those two stages, but it's a good programming practice to attach them close to the phase that they manipulate as it makes clearer their intent. + +For a detailed description of the pipeline hook API, you may refer to the :ref:`pipeline-hooks` guide. + +Disabling pipeline hooks +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.2 + +Any pipeline hook can be disabled from the command line using the :option:`--disable-hook` command line option. +This can be useful to temporarily disable a functionality of the test, e.g., a workaround. + +You can view the list of all the hooks of a test using the :option:`--describe` option: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables.py --describe | jq .[].pipeline_hooks + +.. code-block:: json + + { + "post_setup": [ + "set_executable" + ], + "pre_run": [ + "set_num_threads" + ] + } + + +We could disable the :obj:`set_num_threads` hook by passing ``--disable-hook=set_num_threads``: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables.py --disable-hook=set_num_threads --describe | jq .[].pipeline_hooks + +.. code-block:: json + + { + "post_setup": [ + "set_executable" + ] + } + +The :option:`--disable-hook` option can be passed multiple times to disable multiple hooks at the same time. + + +Environment features and extras +------------------------------- + +We have shown already in the first example the use of *features* in system partition and environment definitions in the configuration file. +These can be used in the :attr:`valid_systems` and :attr:`valid_prog_environs` specifications to help ReFrame pick the right system/environment combinations to run the test. + +In addition to features, the configuration of a system partition or environment can include extra properties that can be accessed from the test and also be used as constraints in the :attr:`valid_systems` and :attr:`valid_prog_environs`. +The following shows the use of extras in the ``baseline_environs.py`` file for defining the compiler flag that enables OpenMP compilation: + +.. literalinclude:: ../examples/tutorial/config/baseline_environs.py + :caption: + :lines: 23-42 + +The :attr:`~config.environments.extras` is a simple key/value dictionary, where the values can have any type and are accessible in the test through the :attr:`current_environ` property as shown in the example above. + + +Execution policies +------------------ + +Having explained the key concepts behind compiled tests as well as the test pipeline, it's time to run our updated test. +However, there is still a small tweak that we need to introduce. + +ReFrame executes tests concurrently. +More precisely, the "compile" and "run" stages of a test execute asynchronously and ReFrame will schedule other tests for running. +Once any of those stages finishes, it will resume execution of the test. +However, this is problematic for our local benchmarks since ReFrame would schedule the GNU-based and the Clang-based tests concurrently and therefore the tests would exhibit lower performance. +For this reason, we will force ReFrame to execute the tests serially with ``--exec-policy=serial``: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_build_run.py --exec-policy=serial -r + +.. code-block:: console + + [----------] start processing checks + [ RUN ] stream_build_test /6c084d40 @tutorialsys:default+gnu-11.4.0 + [ OK ] (1/2) stream_build_test /6c084d40 @tutorialsys:default+gnu-11.4.0 + P: copy_bw: 22273.9 MB/s (r:0, l:None, u:None) + P: triad_bw: 16492.8 MB/s (r:0, l:None, u:None) + [ RUN ] stream_build_test /6c084d40 @tutorialsys:default+clang-14.0.0 + [ OK ] (2/2) stream_build_test /6c084d40 @tutorialsys:default+clang-14.0.0 + P: copy_bw: 22747.9 MB/s (r:0, l:None, u:None) + P: triad_bw: 16541.7 MB/s (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 2/2 test case(s) from 1 check(s) (0 failure(s), 0 skipped, 0 aborted) + + +.. _test-fixtures: + +Test fixtures +============= + +Often a test needs some preparation to be done before it runs, but this preparation may need to be run only once per system partition or environment and not every time the test is run. +A typical example is when we want to build the code of the test once per environment and reuse the executable in multiple different tests. +We can achieve this in ReFrame using *test fixtures*. +Test fixtures are normal ReFrame tests as any other, but they have a scope associated with them and can be fully accessed by the tests that define them. +When test ``A`` is a fixture of test ``B``, then ``A`` will run before ``B`` and ``B`` will have access not only to anything that ``A`` produced, but also to every of its attributes. + +Let's see fixtures in practice by separating our compile-and-run STREAM version into two tests: +a compile-only test that simply builds the benchmark and a run-only version that uses the former as a fixture. + +.. literalinclude:: ../examples/tutorial/stream/stream_fixtures.py + :caption: + :lines: 5- + + +A test fixture is defined with the :func:`~reframe.core.builtins.fixture` builtin: + +.. literalinclude:: ../examples/tutorial/stream/stream_fixtures.py + :caption: + :lines: 25 + +The first argument is a standard ReFrame test which encompasses the fixture logic and will be executed before the current test. +Note that there is no need to decorate a fixture with :func:`@simple_test ` as it will run anyway as part of the test that is using it. +You could still decorate it, though, if you would like to run it independently. + +Each fixture is associated with a scope, which will determine when it will run. +There are the following scopes available: + +- ``session``: The fixture will run once for the whole run session. +- ``partition``: The fixture will run once per system partition. +- ``environment``: The fixture will run once per system partition and environment combination. +- ``test``: The fixture will run every time that the calling test is run. + +Finally, the :func:`~reframe.core.builtins.fixture` builtin returns a handle which can be used to access the target test once it has finished. +This can only be done after the "setup" stage of the current test. +Any attribute of the target test can be accessed through the fixture handle and, in our example, we use the target's test :attr:`stagedir` to construct the final executable. + +.. note:: + + Compile-only tests do not require a validation check, since the test will fail anyway if the compilation fails. + But if one is provided, it will be used. + +Before running the new test, let's try to list it first: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_fixtures.py -l + +.. code-block:: console + + [List of matched checks] + - stream_test /2e15a047 + ^build_stream ~tutorialsys:default+gnu-11.4.0 'stream_binary /2ed36672 + ^build_stream ~tutorialsys:default+clang-14.0.0 'stream_binary /d19d2d86 + Found 1 check(s) + +We will describe in more detail the listing output later in this tutorial, but at the moment it is enough to show that it gives us all the essential information about the test fixtures: their scope and the test variable that they are bound to. +Note also that due to the ``environment`` scope, a separate fixture is created for every environment that will be tested. + +We can now run the benchmarks in parallel to demonstrate that the execution order of the fixtures is respected: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_fixtures.py -r + +.. code-block:: console + + [----------] start processing checks + [ RUN ] build_stream ~tutorialsys:default+gnu-11.4.0 /2ed36672 @tutorialsys:default+gnu-11.4.0 + [ RUN ] build_stream ~tutorialsys:default+clang-14.0.0 /d19d2d86 @tutorialsys:default+clang-14.0.0 + [ OK ] (1/4) build_stream ~tutorialsys:default+gnu-11.4.0 /2ed36672 @tutorialsys:default+gnu-11.4.0 + [ OK ] (2/4) build_stream ~tutorialsys:default+clang-14.0.0 /d19d2d86 @tutorialsys:default+clang-14.0.0 + [ RUN ] stream_test /2e15a047 @tutorialsys:default+gnu-11.4.0 + [ RUN ] stream_test /2e15a047 @tutorialsys:default+clang-14.0.0 + [ OK ] (3/4) stream_test /2e15a047 @tutorialsys:default+gnu-11.4.0 + P: copy_bw: 8182.4 MB/s (r:0, l:None, u:None) + P: triad_bw: 9174.3 MB/s (r:0, l:None, u:None) + [ OK ] (4/4) stream_test /2e15a047 @tutorialsys:default+clang-14.0.0 + P: copy_bw: 7974.4 MB/s (r:0, l:None, u:None) + P: triad_bw: 18494.1 MB/s (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 4/4 test case(s) from 3 check(s) (0 failure(s), 0 skipped, 0 aborted) + +Note that the two STREAM tests are still independent to each other, so they run in parallel and thus the lower performance. + +We will cover more aspects of the fixtures in the following sections, but you are advised to read the API docs of :func:`~reframe.core.builtins.fixture` for a detailed description of all their capabilities. + + +Test variables +============== + +Tests can define *variables* that can be set from the command line. +These are essentially knobs that allow you to change the test's behaviour on-the-fly. +All the test's pre-defined attributes that we have seen so far are defined as variables. +A test variable is defined with the :func:`~reframe.core.builtins.variable` builtin. +Let's augment our STREAM example by adding a variable to control the number of threads to use. + +.. literalinclude:: ../examples/tutorial/stream/stream_variables.py + :caption: + :lines: 5- + +We define a new test variable with the following line: + +.. literalinclude:: ../examples/tutorial/stream/stream_variables.py + :caption: + :lines: 26 + +Variables are typed and any attempt to assign them a value of different type will cause a :class:`TypeError`. +Variables can also have a default value as in this case, which is set to ``0``. +If a variable is not given a value is considered undefined. +Any attempt to read an undefined variable will cause an error. +It is not necessary for a variable to be assigned a value along with its declaration; +this can happen anytime before it is accessed. +Variables are also inherited, that's why we can set the standard variables of a ReFrame test, such as the :attr:`valid_systems`, :attr:`valid_prog_environs` etc., in our subclasses. + +Variables are accessed inside the test as normal class attributes. +In our example, we use the :attr:`num_threads` variable to set the ``OMP_NUM_THREADS`` environment variable accordingly. + +.. literalinclude:: ../examples/tutorial/stream/stream_variables.py + :caption: + :lines: 32-35 + +Variables can be set from the command-line using the :option:`-S` option as ``-S var=value``: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables.py -S num_threads=2 -r + +We will not list the command output here, but you could verify that the variable was set by inspecting the generated run script: + +.. code-block:: bash + :caption: Run in the single-node container. + + cat output/tutorialsys/default/clang-14.0.0/stream_test/rfm_job.sh + +.. code-block:: bash + + #!/bin/bash + export OMP_NUM_THREADS=2 + /home/user/reframe-examples/tutorial/stage/tutorialsys/default/clang-14.0.0/build_stream_d19d2d86/stream.x + +Another thing to notice in the output is the following warning: + +.. code-block:: console + + WARNING: test 'build_stream': the following variables were not set: 'num_threads' + + +When setting a variable as ``-S var=value``, ReFrame will try to set it on all the selected tests, including any fixtures. +If the requested variable is not part of the test, the above warning will be issued. +You can scope the variable assignment in the command line by prefixing the variable name with test's name as follows: ``-S stream_test.num_threads=2``. +In this case, the :attr:`num_threads` variable will be set only in the :class:`stream_test` test. + + +Setting variables in fixtures +----------------------------- + +As we have already mentioned, fixtures are normal ReFrame tests, so they can also define their own variables. +In our example, it makes sense to define a variable in the :class:`build_stream` fixture to control the size of the arrays involved in the computation. +Here is the updated :class:`build_stream` fixture: + +.. literalinclude:: ../examples/tutorial/stream/stream_variables_fixtures.py + :caption: + :pyobject: build_stream + +.. note:: + The :attr:`~reframe.core.buildsystems.BuildSystem.cppflags` attribute of build system refers to the preprocessor flags and not the C++ flags, which are :attr:`~reframe.core.buildsystems.BuildSystem.cxxflags` instead. + +We can set the :attr:`array_size` variable inside the build fixture of our final test through the fixture handle (remember that the fixture handle name is printed in the test listing). +Here is an example: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables_fixtures.py --exec-policy=serial -S stream_test.stream_binary.array_size=50000000 -r + +If you check the generated build script, you will notice the emitted ``-D`` flag: + +.. code-block:: bash + :caption: Run in the single-node container. + + cat output/tutorialsys/default/clang-14.0.0/build_stream_d19d2d86/rfm_build.sh + +.. code-block:: bash + + #!/bin/bash + + _onerror() + { + exitcode=$? + echo "-reframe: command \`$BASH_COMMAND' failed (exit code: $exitcode)" + exit $exitcode + } + + trap _onerror ERR + + clang -DARRAY_SIZE=50000000 -O3 -fopenmp stream.c -o ./stream.x + + + +Test parameterization +===================== + +It is often the case that we want to test different variants of the same test, such as varying the number of tasks in order to perform a scaling analysis on a parallel program. +ReFrame offers a powerful multi-dimensional test parameterization mechanism that automatically generate variants of your tests with different parameter values. +Let's elaborate on this using the STREAM example. +Suppose we want to scale over the number of threads and also try different thread placements. +Here is the updated parameterized :class:`stream_test`: + +.. literalinclude:: ../examples/tutorial/stream/stream_parameters.py + :caption: + :pyobject: stream_test + +Parameters are defined in ReFrame using the :func:`~reframe.core.builtins.parameter` builtin. +This builtin takes simply a list of values for the parameter being defined. +Each parameter is independent and defines a new dimension in the parameterization space. +Parameters can also be inherit and filtered from base classes. +For each point in the final parameterization space, ReFrame will instantiate a different test. +In our example, we expect 12 :class:`stream_test` variants. +Given that we have two valid programming environments and a build fixture with an environment scope, we expect ReFrame to generate and run 26 tests in total (including the fixtures): + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_parameters.py --exec-policy=serial -r + +.. code-block:: console + + [----------] start processing checks + [ RUN ] build_stream ~tutorialsys:default+gnu-11.4.0 /2ed36672 @tutorialsys:default+gnu-11.4.0 + [ OK ] ( 1/26) build_stream ~tutorialsys:default+gnu-11.4.0 /2ed36672 @tutorialsys:default+gnu-11.4.0 + [ RUN ] build_stream ~tutorialsys:default+clang-14.0.0 /d19d2d86 @tutorialsys:default+clang-14.0.0 + [ OK ] ( 2/26) build_stream ~tutorialsys:default+clang-14.0.0 /d19d2d86 @tutorialsys:default+clang-14.0.0 + [ RUN ] stream_test %num_threads=8 %thread_placement=spread /3c8af82c @tutorialsys:default+gnu-11.4.0 + [ OK ] ( 3/26) stream_test %num_threads=8 %thread_placement=spread /3c8af82c @tutorialsys:default+gnu-11.4.0 + P: copy_bw: 24020.6 MB/s (r:0, l:None, u:None) + P: triad_bw: 15453.1 MB/s (r:0, l:None, u:None) + <...omitted...> + [----------] all spawned checks have finished + + [ PASSED ] Ran 26/26 test case(s) from 14 check(s) (0 failure(s), 0 skipped, 0 aborted) + + +Note how the fixture mechanism of ReFrame prevents the recompilation of the STREAM's source code in every test variant: +The source code is only compiled once per toolchain. + + +Parameterizing existing test variables +-------------------------------------- + +We can also parameterize a test on any of its existing variables directly from the command line using the :option:`-P` option. +For example, we could parameterize the STREAM version in ``stream_variables_fixtures.py`` on :attr:`num_threads` as follows: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_variables_fixtures.py -P num_threads=1,2,4,8 --exec-policy=serial -r + + +Parameterizing a fixture +------------------------ + +Fixtures can also be parameterized. +In this case the tests that use them are also parameterized implicitly. +Let's see an example, by parameterizing the build fixture of the STREAM benchmark by adding parameter about the element type (``float`` or ``double``): + +.. literalinclude:: ../examples/tutorial/stream/stream_parameters_fixtures.py + :caption: + :pyobject: build_stream + +As expected, parameters in fixtures are no different than parameters in normal test. +The difference is when you try to list/run the final :class:`stream_test`, where now we have twice as many variants: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -C config/baseline_environs.py -c stream/stream_parameters_fixtures.py -l + + +.. code-block:: console + + - stream_test %num_threads=8 %thread_placement=spread %stream_binary.elem_type=double /ffbd00f1 + ^build_stream %elem_type=double ~tutorialsys:default+gnu-11.4.0 'stream_binary /099a4f75 + ^build_stream %elem_type=double ~tutorialsys:default+clang-14.0.0 'stream_binary /7bd4e3bb + <...omitted...> + - stream_test %num_threads=1 %thread_placement=close %stream_binary.elem_type=float /bc1f32c2 + ^build_stream %elem_type=float ~tutorialsys:default+gnu-11.4.0 'stream_binary /2ed36672 + ^build_stream %elem_type=float ~tutorialsys:default+clang-14.0.0 'stream_binary /d19d2d86 + Found 24 check(s) + +Note, that the test variant name now contains the parameter coming from the fixture. +In total, 52 test cases (24 tests x 2 environments + 2 fixtures x 2 environments) will be run from this simple combination of parameterized tests! + + +Pruning the parameterization space +---------------------------------- + +Sometimes parameters are not independent of each other, as a result some parameter combination may be invalid for the test at hand. +There are two ways to overcome this: + +a. Skip the test if the parameter combination is invalid. +b. Use *parameter packs*. + +Let's see those two methods in practice with a fictitious test. +The first method define two parameters and uses the :func:`skip_if` test method to skip the test if the two parameters have the same value. +The test will be skipped just after it is initialized and the message supplied will be printed as a warning. + +.. literalinclude:: ../examples/tutorial/dummy/params.py + :caption: + :pyobject: echo_test_v0 + +The second method uses a single parameter that packs the valid combinations of the ``x`` and ``y`` parameters. +Note that we also use the optional ``fmt`` argument to provide a more compact formatting for the combined parameter. +Instead of the skip hook, we simply unpack the combined parameters in the :func:`set_executable_opts` hook. + +.. literalinclude:: ../examples/tutorial/dummy/params.py + :caption: + :pyobject: echo_test_v1 + +The advantage of using parameter packs instead of skipping explicitly the test is that we do not get a warning message and the test is more compact. + +.. note:: + + In these tests, we also introduced two more utility functions used in sanity checking, the :func:`~reframe.utility.sanity.and_`, which performs a logical AND of its arguments, and the :func:`~reframe.utility.sanity.assert_eq`, which asserts that its both arguments are equal. + We could have simply written ``return x == self.x and y == self.y`` and the test would still validate, but the utility functions provide more context in case of validation errors. + In fact, we could also provide a custom message to be printed in case of errors, which can be helpful in real case scenarios. + + +Mastering sanity and performance checking +========================================= + +The sanity and performance checking in our STREAM example are simple, but they do represent the most commonly used patterns. +There are cases, however, where we would need a more elaborate sanity checking or extracting the performance measure would not be so straightforward. +The sanity and performance functions (see :func:`@sanity_function ` and :func:`@performance_function `) allow us to write arbitrary code to perform the task at hand, but there are a couple of things to keep in mind: + +- Both sanity and performance functions execute from the test's stage directory. + All relative paths will be resolved against it. +- A sanity function must return a boolean or raise a :class:`~reframe.core.exceptions.SanityError` with a message. + Raising a :class:`~reframe.core.exceptions.SanityError` is the preferred way to denote sanity error and this is exactly what the utility :doc:`sanity functions ` do. +- A performance function must return the value of the extracted figure of merit or raise a :class:`~reframe.core.exceptions.SanityError` in case this is not possible. + + +Understanding the builtin sanity functions +------------------------------------------ + +All the utility functions provided by the framework for sanity checking and the :attr:`stdout` and :attr:`stderr` test attributes are lazily evaluated: +when you call these functions or access these attributes, you are not getting their final value, but instead a special object, named *deferred expression*, which is similar in concept to a *future* or *promise*. +You can include these objects in arbitrary expressions and a new deferred expression will be produced. +In fact, both sanity and performance functions can return a deferred expression, which would return a boolean when evaluated. +And this is what our STREAM sanity and performance functions actually return. + +A deferred expression can be evaluated explicitly by calling its :func:`evaluate` method or pass it to the :func:`~reframe.core.utility.sanity.evaluate` utility function. +For example, to retrieve the actual :attr:`stdout` value, we should do ``self.stdout.evaluate()`` or ``sn.evaluate(self.stdout)``. +Deferred expressions are evaluated implicitly in the following situations: + +1. When trying to iterate over them in ``for`` loop. +2. When trying to include them in an ``if`` expression. +3. When calling :func:`str` on them. + +The ":doc:`deferrables`" page contains details about the underlying mechanism of deferred expressions and gives also concrete examples. + +.. tip:: + + If you are in doubt about the evaluation of a deferred expression, always call :func:`evaluate()` on it. + At the point where the test's :func:`@sanity_function ` is called, all test's attributes are safe to access. + + +.. note:: + + Why deferred expressions? + + In ReFrame versions prior to 3.7, the sanity and performance checking were defined using the :attr:`sanity_patterns` :attr:`perf_patterns` expressions at test's initialization. + In this case, a lazily evaluated expression was necessary since the test has not yet been executed. + The use of :attr:`sanity_patterns` and :attr:`perf_patterns` attributes is still valid today, but it may be deprecated in the future. + + + +Interacting with workload managers +================================== + +ReFrame integrates with many HPC workload managers (batch job schedulers), including Slurm, PBS Pro, Torque and others. +The complete list of scheduler backend can be found `here `__. +Tests in ReFrame are scheduler-agnostic in that they do not need to include any scheduler-specific information. +Instead, schedulers are associated to system partitions. +Each system partition in the configuration file defines the scheduler backend to use along with any scheduler-specific options that are needed to grant access to the desired nodes. + +HPC systems also come with parallel program launchers which are responsible for launching parallel programs onto multiple nodes. +ReFrame supports all major `parallel launchers `__ and allows users to easily define their own custom ones. +Similarly to the batch job schedulers, each system partition is associated to a parallel launcher, which will be used to launch the test's :attr:`executable`. + +In the following, we define a configuration for the Slurm-based pseudo cluster of the tutorial. +We will focus only on the new system configuration as the rest of the configuration remains the same. + +.. literalinclude:: ../examples/tutorial/config/cluster.py + :caption: + :lines: 22-43 + +We define two partitions, one named ``login`` where we are running tests locally (emulating the login nodes of an HPC cluster) and another one named ``compute`` (emulating the compute nodes of an HPC cluster), where we will be submitting test jobs with Slurm and ``srun``. +We use the ``squeue`` scheduler backend, because our Slurm installation does not have job accounting, so we instruct ReFrame to use the ``squeue`` command for querying the job state. +If your Slurm installation has job accounting enabled, you should prefer the ``slurm`` backend, which uses the ``sacct`` for retrieving the job state, which is more reliable. + +Another important parameter is :attr:`~config.systems.partitions.access`, which denotes the job scheduler options needed to access the desired nodes. +In our example, it is redundant to define it as the ``all`` partition is the default, but in most real cases, you will have to define the :attr:`~config.systems.partitions.access` options. + +Let's run our STREAM example with the new configuration: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster.py -c stream/stream_variables_fixtures.py -r + +.. code-block:: console + + [----------] start processing checks + [ RUN ] build_stream ~pseudo-cluster:login+gnu /c5e9e6a0 @pseudo-cluster:login+gnu + [ RUN ] build_stream ~pseudo-cluster:login+clang /d0622327 @pseudo-cluster:login+clang + [ RUN ] build_stream ~pseudo-cluster:compute+gnu /3f5dbfe2 @pseudo-cluster:compute+gnu + [ RUN ] build_stream ~pseudo-cluster:compute+clang /78c4801e @pseudo-cluster:compute+clang + [ OK ] (1/8) build_stream ~pseudo-cluster:login+gnu /c5e9e6a0 @pseudo-cluster:login+gnu + [ OK ] (2/8) build_stream ~pseudo-cluster:login+clang /d0622327 @pseudo-cluster:login+clang + [ OK ] (3/8) build_stream ~pseudo-cluster:compute+gnu /3f5dbfe2 @pseudo-cluster:compute+gnu + [ OK ] (4/8) build_stream ~pseudo-cluster:compute+clang /78c4801e @pseudo-cluster:compute+clang + [ RUN ] stream_test /2e15a047 @pseudo-cluster:login+gnu + [ RUN ] stream_test /2e15a047 @pseudo-cluster:login+clang + [ RUN ] stream_test /2e15a047 @pseudo-cluster:compute+gnu + [ RUN ] stream_test /2e15a047 @pseudo-cluster:compute+clang + [ OK ] (5/8) stream_test /2e15a047 @pseudo-cluster:login+gnu + P: copy_bw: 9062.2 MB/s (r:0, l:None, u:None) + P: triad_bw: 8344.9 MB/s (r:0, l:None, u:None) + [ OK ] (6/8) stream_test /2e15a047 @pseudo-cluster:login+clang + P: copy_bw: 25823.0 MB/s (r:0, l:None, u:None) + P: triad_bw: 12732.2 MB/s (r:0, l:None, u:None) + [ OK ] (7/8) stream_test /2e15a047 @pseudo-cluster:compute+clang + P: copy_bw: 11215.5 MB/s (r:0, l:None, u:None) + P: triad_bw: 7960.5 MB/s (r:0, l:None, u:None) + [ OK ] (8/8) stream_test /2e15a047 @pseudo-cluster:compute+gnu + P: copy_bw: 10300.7 MB/s (r:0, l:None, u:None) + P: triad_bw: 9647.1 MB/s (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 8/8 test case(s) from 5 check(s) (0 failure(s), 0 skipped, 0 aborted) + + +Note how the test runs every each partition and environment combination. +For the ``login`` partition the generated script is the same as for the local execution, whereas for the ``compute`` partition ReFrame generates a job script, which submits with ``sbatch``: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + cat /scratch/rfm-stage/output/pseudo-cluster/compute/gnu/stream_test/rfm_job.sh + +.. code-block:: bash + + #!/bin/bash + #SBATCH --job-name="rfm_stream_test" + #SBATCH --ntasks=1 + #SBATCH --output=rfm_job.out + #SBATCH --error=rfm_job.err + #SBATCH -p all + srun /scratch/rfm-stage/stage/pseudo-cluster/compute/gnu/build_stream_3f5dbfe2/stream.x + + +You may have noticed that we use the :option:`--prefix` option when running ReFrame this time. +This option changes the prefix of the stage and output directory. +All scheduler backends, except ``ssh``, require the test's stage directory to be shared across the local and remote nodes, therefore set it to point under the shared ``/scratch`` volume. + +.. note:: + For running the Slurm-based examples, make sure to follow the instructions in :ref:`multi-node-setup` for bringing up and accessing this cluster. + + +Selecting specific partitions or environments to run +---------------------------------------------------- + +ReFrame can generated many test cases if have many partitions and environments and you will most likely need to scope down the test space. +You could use the :option:`--system` and :option:`-p` options to restrict a test to a single partition and/or a single environment. +To run only the GCC tests on the compute partition you could do the following: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster.py -c stream/stream_variables_fixtures.py \ + --system=pseudo-cluster:compute -p gnu -r + + +Compiling remotely +------------------ + +By default, ReFrame compiles the test's source code locally on the node that it runs. +This may be problematic in cases that cross-compilation is not possible and the test's code needs to be compiled on the remote nodes. +This can be achieved by settings the test's :attr:`build_locally` attribute to :obj:`False` with ``-S build_locally=0``. +In this case, ReFrame will generate a job script also for the compilation job and submit it for execution. + +Passing additional scheduler options +------------------------------------ + +There are two ways to pass additional options to the backend scheduler: either by modifying the :attr:`~reframe.core.schedulers.Job` instance associated to the test or by defining an extra resource at the partition configuration requesting this from the test. +Let's see both methods: + +Modifying the test job's options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This method is quite straightforward: +you need simply to define a pre-run hook and set the ``self.job.options``. +For example, to pass the ``--mem`` Slurm option to the job submitted by the test, you could do the following: + +.. code-block:: python + + @run_before('run') + def set_mem_constraint(self): + self.job.options = ['--mem=1000'] + +The advantage of this method is its simplicity, but it injects system-specific information to the test tying it to the system scheduler. +You could make the test more robust, however, by restricting it to system partitions with Slurm by setting the :attr:`valid_systems` accordingly: + +.. code-block:: python + + valid_systems = [r'%scheduler=slurm', r'%scheduler=squeue'] + + +Defining extra scheduler resources +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This method comprises two steps. +First, we need to define a `resource `__ in the partition configuration: + +.. literalinclude:: ../examples/tutorial/config/cluster_resources.py + :caption: + :lines: 41-46 + +Each resource has a name and a list of scheduler options that will be emitted in the job script when this resource will be requested by the test. +The scheduler options specification can contain placeholders that will be filled from the test. + +Now we can use this resource very in the test by settings its :attr:`extra_resources`: + +.. code-block:: python + + extra_resources = { + 'memory': {'size': '1000'} + } + +The advantage of this method is that it is completely scheduler-agnostic. +If the system partition where the test is running on does not define a resource, the request will be ignored. + +Both methods of setting addition job options are valid and you may use whichever of the two fits best your use case. + + +Modifying the launch command +---------------------------- + +Sometimes it's useful to modify the launch command itself by prepending another program, such as debugger or profiler. +You can achieve this by setting the :attr:`~reframe.core.launchers.JobLauncher.modifier` and :attr:`~reframe.core.launchers.JobLauncher.modifier_options` of the test job's launcher: + + +.. code-block:: python + + @run_before('run') + def run_with_gdb(self): + self.job.launcher.modifier = 'gdb' + self.job.launcher.modifier_options = ['-batch', '-ex run', '-ex bt', '--args'] + + + +Replacing the launch command +---------------------------- + +Sometimes you may want to replace completely the launcher associated with the partition that the test will run. +You can do that with the following hook: + +.. code-block:: python + + from reframe.core.backends import getlauncher + ... + + @run_before('run') + def replace_launcher(self): + self.job.launcher = getlauncher('local')() + +The :func:`~reframe.core.backends.getlauncher` utility function returns the type the implements the launcher with the given name. +The supported launcher names are those registered with the framework, i.e., all the names listed +`here `__ as well as any :ref:`user-registered ` launcher. +Once we have the launcher type, we instantiate it and replace the job's launcher. + + +Multiple job steps +------------------ + +A job step is a command launched with the parallel launcher. +ReFrame will only launch the :attr:`executable` as a job step. +You can launch multiple job steps by leveraging the :attr:`prerun_cmds` or :attr:`postrun_cmds` test attributes. +These are commands to be executed before or after the main :attr:`executable` and, normally, they are not job steps: they are simple Bash commands. +However, you can use the :attr:`reframe.core.launcher.JobLauncher` API to emit the parallel launch command and convert them to a job step as shown in the following example: + +.. literalinclude:: ../examples/tutorial/stream/stream_multistep.py + :caption: + :lines: 37-40 + +Here we invoke the job launcher's :func:`~reframe.core.launchers.JobLauncher.run_command` method, which is responsible for emitting the launcher prefix based on the current partition. + + +Generally, ReFrame generates the job shell scripts using the following pattern: + +.. code-block:: bash + + #!/bin/bash -l + {job_scheduler_preamble} + {prepare_cmds} + {env_load_cmds} + {prerun_cmds} + {parallel_launcher} {executable} {executable_opts} + {postrun_cmds} + +The ``job_scheduler_preamble`` contains the backend job scheduler directives that control the job allocation. +The ``prepare_cmds`` are commands that can be emitted before the test environment commands. +These can be specified with the :attr:`~config.systems.partitions.prepare_cmds` partition configuration option. +The ``env_load_cmds`` are the necessary commands for setting up the environment of the test. +These include any modules or environment variables set at the `system partition level `__ or any `modules `__ or `environment variables `__ set at the test level. +Then the commands specified in :attr:`prerun_cmds` follow, while those specified in the :attr:`postrun_cmds` come after the launch of the parallel job. +The parallel launch itself consists of three parts: + +#. The parallel launcher program (e.g., ``srun``, ``mpirun`` etc.) with its options, +#. the test executable as specified in the :attr:`~reframe.core.pipeline.executable` attribute and +#. the options to be passed to the executable as specified in the :attr:`executable_opts` attribute. + + +Accessing CPU topology information +================================== + +.. versionadded:: 3.7 + +Sometimes a test may need to access processor topology information for the partition it runs so as to better set up the run. +Of course, you could hard code the information in the test, but it wouldn't be so portable. +ReFrame auto-detects the local host topology and it can also auto-detect the topology of remote hosts. +It makes available this information to the test through the :attr:`current_partition`'s :attr:`~reframe.core.systems.SystemPartition.processor` attribute. + +Let's use this feature to set the number of threads of our STREAM benchmark to the host's number of cores, if it is not defined otherwise. + +.. literalinclude:: ../examples/tutorial/stream/stream_cpuinfo.py + :caption: + :lines: 32-39 + +Note also the use of the :func:`skip_if_no_procinfo()` function which will cause ReFrame to skip the test if there is no processor information available. + +Let's try running the test on our pseudo-cluster: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster.py -c stream/stream_cpuinfo.py -p gnu -r + +.. code-block:: console + + [==========] Running 3 check(s) + [==========] Started on Mon Feb 12 21:55:54 2024+0000 + + [----------] start processing checks + [ RUN ] build_stream ~pseudo-cluster:login+gnu /c5e9e6a0 @pseudo-cluster:login+gnu + [ RUN ] build_stream ~pseudo-cluster:compute+gnu /3f5dbfe2 @pseudo-cluster:compute+gnu + [ OK ] (1/4) build_stream ~pseudo-cluster:login+gnu /c5e9e6a0 @pseudo-cluster:login+gnu + [ OK ] (2/4) build_stream ~pseudo-cluster:compute+gnu /3f5dbfe2 @pseudo-cluster:compute+gnu + [ RUN ] stream_test /2e15a047 @pseudo-cluster:login+gnu + [ RUN ] stream_test /2e15a047 @pseudo-cluster:compute+gnu + [ SKIP ] (3/4) no topology information found for partition 'pseudo-cluster:compute' + [ OK ] (4/4) stream_test /2e15a047 @pseudo-cluster:login+gnu + P: copy_bw: 36840.6 MB/s (r:0, l:None, u:None) + P: triad_bw: 18338.8 MB/s (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 3/4 test case(s) from 3 check(s) (0 failure(s), 1 skipped, 0 aborted) + + +Indeed, for the ``login`` partition, the generated script contains the correct number of threads: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + cat /scratch/rfm-stage/output/pseudo-cluster/login/gnu/stream_test/rfm_job.sh + +.. code-block:: bash + + #!/bin/bash + export OMP_NUM_THREADS=8 + /scratch/rfm-stage/stage/pseudo-cluster/login/gnu/build_stream_c5e9e6a0/stream.x + +However, if you noticed, the ``compute`` partition was skipped as no topology information was found. +ReFrame by default does not try to auto-detect remote partitions, because this could be time consuming. +To enable remote host auto-detection, we should set the :envvar:`RFM_REMOTE_DETECT` or the equivalent :attr:`~config.general.remote_detect` configuration option. + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + RFM_REMOTE_WORKDIR=/scratch/rfm-stage RFM_REMOTE_DETECT=1 reframe --prefix=/scratch/rfm-stage/ -C config/cluster.py -c stream/stream_cpuinfo.py -p gnu -r + + +.. code-block:: console + + ... + Detecting topology of remote partition 'pseudo-cluster:compute': this may take some time... + ... + [ OK ] (3/4) stream_test /2e15a047 @pseudo-cluster:compute+gnu + P: copy_bw: 19288.6 MB/s (r:0, l:None, u:None) + P: triad_bw: 15243.0 MB/s (r:0, l:None, u:None) + ... + +.. note:: + + In our setup we need to set also the :envvar:`RFM_REMOTE_WORKDIR` since the current volume (``/home``) is not shared with the head node. + +ReFrame caches the result of host auto-detection, so that it avoids re-detecting the topology every time. +For a detailed description of the process, refer to the documentation of the :attr:`~config.system.partitions.processor` configuration option. + + +Device information +------------------ + +ReFrame cannot auto-detect at the moment device information, such as attached accelerators, NICs etc. +You can however add manually in the configuration any interesting device and this will be accessible from inside the test through the :attr:`current_partition`. +For more information check the documentation of the :attr:`~config.systems.partitions.devices` configuration parameter. + + +.. _multi-node-tests: + +Multi-node tests +================ + +Multi-node tests are quite straightforward in ReFrame. +All you need is to specify the task setup and the scheduler backend and parallel launcher will emit the right options. + +The following tests run download, compile and launch the `OSU benchmarks `__. + +.. literalinclude:: ../examples/tutorial/mpi/osu.py + :caption: + :lines: 5- + +Notice the assignment of :attr:`num_tasks` and :attr:`num_tasks_per_node` in the base test class :class:`osu_base_test`. +The :class:`RegressionTest` base class offers many more attributes for specifying the placement of tasks on the nodes. + +Unrelated to their multi-node nature, these examples showcase some other interesting aspects of ReFrame tests: + +- Fixtures can use other fixtures. +- The ``session`` scope of the :attr:`osu_benchmarks` fixture will make the :class:`fetch_osu_benchmarks` test that downloads the benchmarks to run only once at the beginning of the session. + Similarly, the ``environment`` scope of the :attr:`osu_binaries` fixture will make the :class:`build_osu_benchmarks` test execute once per partition and environment combination. +- Instead of using the :func:`@performance_function ` decorator to define performance variables, we could directly set the :attr:`perf_variables` test attribute. + This is useful when we want to programmatically generate test's performance variables. + +Here is how to execute the tests. +Note that we are using another configuration file, which defines an MPI-enabled environment so that we can compile the OSU benchmarks: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster_mpi.py -c mpi/osu.py --exec-policy=serial -r + + +.. code-block:: console + + [----------] start processing checks + [ RUN ] fetch_osu_benchmarks ~pseudo-cluster /d20db00e @pseudo-cluster:compute+gnu-mpi + [ OK ] (1/5) fetch_osu_benchmarks ~pseudo-cluster /d20db00e @pseudo-cluster:compute+gnu-mpi + [ RUN ] build_osu_benchmarks ~pseudo-cluster:compute+gnu-mpi /be044b23 @pseudo-cluster:compute+gnu-mpi + [ OK ] (2/5) build_osu_benchmarks ~pseudo-cluster:compute+gnu-mpi /be044b23 @pseudo-cluster:compute+gnu-mpi + [ RUN ] osu_allreduce_test /63dd518c @pseudo-cluster:compute+gnu-mpi + [ OK ] (3/5) osu_allreduce_test /63dd518c @pseudo-cluster:compute+gnu-mpi + P: bandwidth: 38618.05 MB/s (r:0, l:None, u:None) + [ RUN ] osu_bandwidth_test /026711a1 @pseudo-cluster:compute+gnu-mpi + [ OK ] (4/5) osu_bandwidth_test /026711a1 @pseudo-cluster:compute+gnu-mpi + P: bandwidth: 144.96 MB/s (r:0, l:None, u:None) + [ RUN ] osu_latency_test /d2c978ad @pseudo-cluster:compute+gnu-mpi + [ OK ] (5/5) osu_latency_test /d2c978ad @pseudo-cluster:compute+gnu-mpi + P: latency: 12977.31 us (r:0, l:None, u:None) + [----------] all spawned checks have finished + + [ PASSED ] Ran 5/5 test case(s) from 5 check(s) (0 failure(s), 0 skipped, 0 aborted) + + +.. note:: + + The parameters passed to the OSU benchmarks are adapted for the purposes of the tutorial. + You should adapt them if running on an actual parallel cluster. + + +Managing the run session +======================== + +ReFrame offers a rich command line interface that allows users to manage and execute their test suite. +In this section, we will briefly discuss the most important command line options. +For a complete reference, users are referred to the :doc:`manpage`. + +Test listing +------------ + +This is probably the most important action after the test execution. +We have seen already the :option:`-l` option that performs the listing of the tests to be executed. +It is always a good practice to first list the tests to run before executing them in order to avoid surprises. +By default, the :option:`-l` lists only the tests to be executed: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C config/cluster.py -c stream/stream_build_run.py -l + + +.. code-block:: console + + [List of matched checks] + - stream_build_test /6c084d40 + Found 1 check(s) + +However, on a system with multiple partitions and environments, the test will run on all the supported combinations. +In ReFrame's terminology, these are called *test cases*. +You can instruct the :option:`-l` to list the actual (concretized) test cases that will eventually run with ``-lC``: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C config/cluster.py -c stream/stream_build_run.py -lC + + +.. code-block:: console + + [List of matched checks] + - stream_build_test /6c084d40 @pseudo-cluster:login+gnu + - stream_build_test /6c084d40 @pseudo-cluster:login+clang + - stream_build_test /6c084d40 @pseudo-cluster:compute+gnu + - stream_build_test /6c084d40 @pseudo-cluster:compute+clang + Concretized 4 test case(s) + +Notice the ``@pseudo-cluster:login+gnu`` notation that is appended to each test case: +this is the exact combination of partition and environment that the test will run for. + +You can also opt for a detailed listing with the :option:`-L` option, which also accepts the ``C`` argument for producing the concretized test cases. + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C config/cluster.py -c stream/stream_build_run.py -LC + + +.. code-block:: console + + [List of matched checks] + - stream_build_test /6c084d40 @pseudo-cluster:login+gnu [variant: 0, file: '/home/admin/reframe-examples/tutorial/stream/stream_build_run.py'] + - stream_build_test /6c084d40 @pseudo-cluster:login+clang [variant: 0, file: '/home/admin/reframe-examples/tutorial/stream/stream_build_run.py'] + - stream_build_test /6c084d40 @pseudo-cluster:compute+gnu [variant: 0, file: '/home/admin/reframe-examples/tutorial/stream/stream_build_run.py'] + - stream_build_test /6c084d40 @pseudo-cluster:compute+clang [variant: 0, file: '/home/admin/reframe-examples/tutorial/stream/stream_build_run.py'] + Concretized 4 test case(s) + +This listing prints the test variant (each parameter value in a parameterized test generates a new variant) and the file from where this test was loaded. + +There are several parts and symbols in a full test case listing. +The following figure explains them all in detail. + +.. figure:: _static/img/test-naming.svg + :align: center + + :sub:`Test naming scheme.` + + +Test discovery +-------------- + +This is the phase that ReFrame looks for tests and loads them. +By default, it looks inside the ``./checks`` directory unless the :option:`-c` is specified. +We have used already this option to load the tests from specific files. +The :option:`-c` can be used multiple times to load more files or its argument may be a directory, in which case all the ReFrame test files found will be loaded. +Note that ReFrame will refuse to load test with the same name. +Finally, this option may be combined with the :option:`-R` to recursively descend inside a directory. + + +Test filtering +-------------- + +ReFrame offers several options for filtering the tests loaded during test discovery. +We have seen already the :option:`-p` option to select tests that support a specific environment. + +Perhaps the most important filtering option is the :option:`-n` option, which filters tests by name. +Its argument can take different forms that help in different scenarios: + +- It can be a regular expression that will be searched inside the full test's name, including the parameters. + For example, in order to select only the test variants that have the ``num_threads`` parameter set to ``1`` in the ``stream/stream_parameters_fixtures.py``, we can do the following: + + .. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C config/cluster.py -c stream/stream_parameters_fixtures.py -l -n '%num_threads=1' + +- It can be of the form ``/`` in which case the exact test with the specified hash will be selected + +- It can be of the form ``test_name@`` in which case a specific variant of a parameterized test will be selected. + +Another useful filtering option is the :option:`-t` option, which selects tests by tag. +You can assign multiple tags to a test by settings its :attr:`tags` attribute. +You can use tags to effectively categorize tests. + +Finally, a powerful selection option is :option:`-E`. +This allows you to filter tests by evaluating an expression over their variables or parameters. +For example, we could select all tests with 4 threads and a ``spread`` placement as follows: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C config/cluster.py -c stream/stream_parameters_fixtures.py -l -E 'num_threads==4 and thread_placement=="spread"' + +The syntax of the expression must be valid Python. + + +Execution modes +--------------- + +ReFrame allows you to group command-line options into so called *execution modes*. +Execution mode are defined in the :attr:`~config.modes` configuration section and are merely a collection of command-line options that will be passed to ReFrame when the mode is selected with the :option:`--mode` option. +Here is an example: + +.. literalinclude:: ../examples/tutorial/config/cluster.py + :caption: + :lines: 65-70 + +We can now select this mode with ``--mode=singlethread`` and get only the tests where ``num_threads=1``. +Obviously, modes become more useful when we need to abbreviate many options. + + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C config/cluster.py -c stream/stream_parameters_fixtures.py --mode=singlethread -l + +You can use any ReFrame option in an execution mode except the :option:`-C` and :option:`--system` options, since these determine the configuration file and the system configuration to load. + +Retrying tests +-------------- + +You can instruct ReFrame to retry the failing tests with the :option:`--max-retries` option. +The retries will happen at the end of the session and not immediately after the test fails. +Each retried test will be staged in a separate directory. +If the test passed in retries, its result is "success" with a mention that it has passed in retries. + +The :option:`--max-retries` has an effect only in the current run session. +However, in order to rerun the failed tests of a previous session, you should use the :option:`--restore-session` :option:`--failed` options. +This is particularly useful when a failed test has long-running dependencies that have succeeded in the previous run. +In this case, the dependencies will be restored and only the failed tests will be rerun. + +Let's see an artificial example that uses the following test dependency graph. + +.. _fig-deps-complex: + +.. figure:: _static/img/deps-complex.svg + :align: center + + :sub:`Complex test dependency graph. Nodes in red are set to fail.` + + +Tests :class:`T2` and :class:`T8` are set to fail. +Let's run the whole test DAG: + +.. code-block:: bash + :caption: Run in the single-node container. + + cd reframe-examples/tutorial/ + reframe -c deps/deps_complex.py -r + +.. Literalinclude:: listings/deps_complex_run.txt + :Language: console + +You can restore the run session and run only the failed test cases as follows: + + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --restore-session --failed -r + + +Of course, as expected, the run will fail again, since these tests were designed to fail. + +Instead of running the failed test cases of a previous run, you might simply want to rerun a specific test. +This has little meaning if you don't use dependencies, because it would be equivalent to running it separately using the :option:`-n` option. +However, if a test was part of a dependency chain, using :option:`--restore-session` will not rerun its dependencies, but it will rather restore them. +This is useful in cases where the test that we want to rerun depends on time-consuming tests. +There is a little tweak, though, for this to work: +you need to have run with :option:`--keep-stage-files` in order to keep the stage directory even for tests that have passed. +This is due to two reasons: +(a) if a test needs resources from its parents, it will look into their stage directories and +(b) ReFrame stores the state of a finished test case inside its stage directory and it will need that state information in order to restore a test case. + +Let's try to rerun the :class:`T6` test from the previous test dependency chain: + + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -c deps/deps_complex.py --keep-stage-files -r + reframe --restore-session --keep-stage-files -n T6 -r + + +Notice how only the :class:`T6` test was rerun and none of its dependencies, since they were simply restored: + +.. literalinclude:: listings/deps_rerun_t6.txt + :language: console + + +If we tried to run :class:`T6` without restoring the session, we would have to rerun also the whole dependency chain, i.e., also :class:`T5`, :class:`T1`, :class:`T4` and :class:`T0`. + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -c deps/deps_complex.py -n T6 -r + +.. literalinclude:: listings/deps_run_t6.txt + :language: console + + +Running continuously +-------------------- + +You can instruct ReFrame to rerun the whole test session multiple times or for a specific duration, using the :option:`--reruns` or :option:`--duration` options. +These options will repeat the session once the first round of tests has finished. +For example, the following command will run the STREAM benchmark repeatedly for 30 minutes: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe -c stream_runonly.py --duration=30m -r + + +Generating tests on-the-fly +--------------------------- + +ReFrame can generate new tests dynamically from the already :ref:`selected ` tests. +We have seen already the :option:`-P` option that parameterizes the selected tests on a specific variable. + +Another very useful test generation option for Slurm-based partitions is the :option:`--distribute` option. +This will distribute the selected tests on all the available idle nodes of their valid system partitions. +A separate test variant will be created for every available node and it will be pinned to it. +It also accepts an optional argument distribute the tests either on all the nodes of the partition regardless of their state or only on the nodes that are in a specific state. +It can be combined with the :option:`-J` option to further restrict the node selection, e.g., ``-J reservation=maint`` to submit to all the nodes in the ``main`` reservation. +The following example will run our STREAM test on all the nodes of our pseudo-cluster: + + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster.py -c stream/stream_fixtures.py -p gnu --system=pseudo-cluster:compute --distribute=all -r + +Note that similarly to the :option:`-P` option, :option:`--distribute` parameterizes the leaf tests and not their fixtures, so the build fixture of the STREAM benchmark will be executed once and only the binary will run on every node, which is the desired behavior in our case. + +By inspecting the generated script files, you will notice that ReFrame emits the ``--nodelist`` to pin the tests to the cluster nodes: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + cat /scratch/rfm-stage/output/pseudo-cluster/compute/gnu/stream_test_pseudo-cluster_compute_8de19aca/rfm_job.sh + +.. code-block:: bash + + #!/bin/bash + #SBATCH --job-name="rfm_stream_test_pseudo-cluster_compute_8de19aca" + #SBATCH --ntasks=1 + #SBATCH --output=rfm_job.out + #SBATCH --error=rfm_job.err + #SBATCH --nodelist=nid01 + #SBATCH -p all + srun /scratch/rfm-stage/stage/pseudo-cluster/compute/gnu/build_stream_3f5dbfe2/stream.x + + +Another useful test generation option is :option:`--repeat`. +This will repeat the selected tests a specified number of times. +Conversely to the :option:`--reruns` option explained before, the repeated tests will be executed concurrently. +In practice, this option clones the selected tests ``N`` times and submits them at once unless the serial execution policy is used. + +Aborting the test session +------------------------- + +ReFrame does careful error handling. +If a test fails due to a programming error or even if a test tries to explicitly call :func:`sys.exit`, ReFrame will simply mark the test as failure and will continue with the rest of the session. + +You can kill the current session gracefully by pressing Ctl-C or sending the ``SIGINT`` signal to the ReFrame process. +In this case, ReFrame will cancel all spawned jobs, either local or remote, so as to avoid spilling resources. + +Another useful option that will finish the session prematurely is :option:`--maxfail`. +This will cause ReFrame to stop after a specified amount of failures. + + +.. _managing-the-configuration: + +Managing the configuration +========================== + +Adding more systems to the ReFrame configuration file will soon make it quite big. +ReFrame can build its final configuration by combining several smaller configuration files. +For example, you could maintain a configuration file per system and keep logging and general settings in a different file. +You can chain the configuration files by passing multiple times the :option:`-C` option. +Alternatively, you can set the :envvar:`RFM_CONFIG_PATH` variable to specify the directories where ReFrame will search for configuration files. +The configuration files in this case must be named as ``settings.py``. + +In the following, we have split the ``cluster_perflogs.py`` in three different configuration files as follows: + +.. code-block:: console + + config/multifile/ + ├── common + │   └── settings.py + ├── environments + │   └── settings.py + └── pseudo-cluster + └── settings.py + +Since the configuration file names are normalized, we could use the :envvar:`RFM_CONFIG_PATH` environment variable instead of the :option:`-C` option: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + export RFM_CONFIG_PATH=$(pwd)/config/multifile/common:$(pwd)/config/multifile/environments:$(pwd)/config/multifile/pseudo-cluster + + +Inspecting the loaded configuration +----------------------------------- + +ReFrame offers the very convenient :option:`--show-config`, that allows you to inspect the actual loaded configuration or query configuration values. +Indeed having set the environment variable :envvar:`RFM_CONFIG_PATH`, running + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --show-config + +will show us the current configuration. +Note that the loaded configuration resolves to the auto-detected system. +Even if we load a configuration file with multiple files, the :option:`--show-config` option will show the configuration of the current system: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C :config/baseline.py --show-config + +Notice that the ``tutorialsys`` was not matched and therefore the current system is the ``generic``. + +.. note:: + + Using the ``:`` before the configuration filename passed to the :option:`-C` option, instructs ReFrame to drop any configuration built so far from the :envvar:`RFM_CONFIG_PATH`. + + +The :option:`--show-config` option takes also an optional argument which will allows you to select a specific configuration parameter: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --show-config=systems/0/name + +.. code-block:: json + + "pseudo-cluster" + +You can also use the :option:`--show-config` option to retrieve the default value of a configuration parameter, even if this is not defined in the configuration file: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --show-config=general/0/trap_job_errors + +.. code-block:: json + + false + + +Scoping configuration options +----------------------------- + +ReFrame allows you to limit the effect of configuration options only to certain systems. +Every top-level configuration object, except :attr:`~config.systems`, has an optional :attr:`config.environment.target_systems` that accepts a list of systems where this object is valid for. +For example, if we wanted to set :attr:`~general.trap_job_errors` only for the ``pseudo-cluster`` system, we could add the following in our configuration: + +.. code-block:: python + + 'general': [ + { + 'trap_job_errors': True, + 'target_systems': ['pseudo-cluster'] + } + ] + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe -C :config/cluster.py --show-config=general/0/trap_job_errors + +.. code-block:: json + + true + + +.. _Logging: + +Logging +======= + +There are two types of logs that ReFrame produces: + +1. Activity logs, which log the activities of the framework and in their detailed version can be useful for debugging. +2. Performance logs, which is about recording the obtained performance metrics of performance tests. + + +Activity logging +---------------- + +By default, ReFrame generates a debug log file in the system's temporary directory. +This is quite a detailed log. +Logging can be configured in the :attr:`~config.logging` section of the configuration file. +Multiple logging handlers can be registered that will log messages to different sinks at different levels. +Let's see an example on how to setup ReFrame to save its output in a ``reframe_.out` and a detailed debug output in ``reframe_.log``: + +.. literalinclude:: ../examples/tutorial/config/cluster_logging.py + :caption: + :lines: 71-93 + + +Controlling output verbosity +---------------------------- + +You may control the output verbosity using :option:`-v` to increase it or :option:`-q` to decrease it. +Both options can be specified multiple times to further increase or decrease verbosity. + +The following table shows the available verbosity levels and the effect of the above options: + +.. csv-table:: ReFrame's verbosity levels + :align: center + :header: Option, Level + + ``-qq``, error + ``-q``, warning + *default*, info + ``-v``, verbose + ``-vv``, debug + ``-vvv``, debug2 + + +Performance logging +------------------- + +We have talked briefly about performance logging in :ref:`run-reports-and-performance-logging` but here we will present in more detail the information displayed and how you can configure it. + +By default, ReFrame stores the performance data obtained from *performance tests* in a CSV file. +A performance test is a test that defines at least one figure of merit (see :ref:`writing-your-first-test`). +The default location of the performance logs is ``/perflogs///.log``, where ```` is the output prefix as specified by the :option:`--prefix` or :option:`--perflogdir` options and ```` refers to the test's class name. +Every time that a variant of the test is run, a new line will be appended to this file. +Here is the performance log file for the ``stream_test`` on our ``pseudo-cluster:compute`` partition: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + cat /scratch/rfm-stage/perflogs/pseudo-cluster/compute/stream_test.log + +.. code-block:: console + + result|job_completion_time|descr|env_vars|environ|exclusive_access|extra_resources|job_completion_time_unix|job_exitcode|job_nodelist|job_submit_time|jobid|modules|name|num_cpus_per_task|num_gpus_per_node|num_tasks|num_tasks_per_core|num_tasks_per_node|num_tasks_per_socket|partition|copy_bw_value|copy_bw_unit|copy_bw_ref|copy_bw_lower_thres|copy_bw_upper_thres|triad_bw_value|triad_bw_unit|triad_bw_ref|triad_bw_lower_thres|triad_bw_upper_thres|short_name|system|unique_name|use_multithreading + pass|2024-02-21T22:51:16||{}|gnu|false|{}|1708555876.746763|null|None|1708555874.6122677|65||stream_test|null|null|1|null|null|null|compute|21116.7|MB/s|0|None|None|14813.0|MB/s|0|None|None|stream_test|pseudo-cluster|stream_test|null + pass|2024-02-21T22:51:19||{}|clang|false|{}|1708555879.4456542|null|None|1708555877.3607173|66||stream_test|null|null|1|null|null|null|compute|18405.7|MB/s|0|None|None|14997.1|MB/s|0|None|None|stream_test|pseudo-cluster|stream_test|null + pass|2024-02-25T20:45:17||{}|gnu|false|{}|1708893917.8675761|null|None|1708893915.3461528|69||stream_test|null|null|1|null|null|null|compute|11429.4|MB/s|0|None|None|10674.8|MB/s|0|None|None|stream_test|pseudo-cluster|stream_test|null + pass|2024-02-25T20:45:17||{}|clang|false|{}|1708893917.9110286|null|None|1708893915.4608803|70||stream_test|null|null|1|null|null|null|compute|11909.9|MB/s|0|None|None|8325.0|MB/s|0|None|None|stream_test|pseudo-cluster|stream_test|null + + +The first line serves as a header. +If ReFrame determines that the header must change due to a change in the test (e.g., new variables, new figure of merits, etc.), it will back up the existing file to ``.log.h0`` and will create a new file to hold the current performance data. + +You may have noticed that ReFrame logs a lot of information along with the test's performance. +The reason for that is to cover a wide range of usage, but you might not be interested in all that information, especially if your test's setup is fixed. +Let's see how we can change the perflogs format for our example: + +.. literalinclude:: ../examples/tutorial/config/cluster_perflogs.py + :caption: + :lines: 71-89 + +The :attr:`~config.logging.handlers_perflog` configuration section defines a list of log handlers where the performance data for every test that finishes will be sent to. +The ``filelog`` handler manages the writing of performance data to files per test as described above. +Let's walk briefly through the most important parts of its configuration: + +- The :attr:`~config.logging.handlers_perflog..filelog..prefix` is an additional directory prefix under the global prefix (see :option:`--prefix` option) where the perflogs will be saved. + The formatting placeholders are described below. +- The :attr:`~config.logging.handlers_perflog.format` specifies how the log record will be formatted. + Each placeholder of the form ``%(placeholder)s`` is replaced by the actual value during runtime. + All placeholders starting with ``check_`` refer to test attributes. + You can check the complete list of supported placeholders in the configuration :attr:`reference guide `. + In this particular example, we will log only the test result, the (formatted) completion time, the list of nodes where the test was run and the obtained values of the test's performance variables. +- The :attr:`~config.logging.handlers_perflog.format_perfvars` specifies how the performance values (the ``%(check_perfvalues)s`` placeholder) will be formatted. + In this case, we will only log the obtained value and its unit. + Note that ReFrame will repeat this pattern for all the performance variables defined in the test and this is why we need to end the pattern with the separator, here the ``,``. + +Let's rerun our STREAM example using the new configuration: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster_perflogs.py -c stream/stream_fixtures.py -r + cat /scratch/rfm-stage/perflogs/pseudo-cluster/compute/stream_test.log + + +.. code-block:: console + + result,job_completion_time,system:partition,environ,copy_bw_value,copy_bw_unit,triad_bw_value,triad_bw_unit + pass,2024-02-26T22:39:52,pseudo-cluster:compute,gnu,11527.4,MB/s,10110.8,MB/s + pass,2024-02-26T22:39:52,pseudo-cluster:compute,clang,11798.5,MB/s,8370.0,MB/s + + +Sending performance data to an HTTP endpoint +-------------------------------------------- + +You can instruct ReFrame to send the performance logs directly to an HTTP endpoint. +An example is sending test performance records to `Elastic `__. +This is handled by the ``httpjson`` handler and an example configuration is the following: + +.. literalinclude:: ../examples/tutorial/config/cluster_perflogs_httpjson.py + :caption: + :lines: 106-119 + +The :attr:`~config.logging.handlers_perflog..httpjson..url` key refers to the service endpoint, the :attr:`~config.logging.handlers_perflog..httpjson..extra_headers` are additional headers to be included in the POST request, whereas the :attr:`~config.logging.handlers_perflog..httpjson..extras` and the :attr:`~config.logging.handlers_perflog..httpjson..ignore_keys` are additional keys to send or keys to exclude, respectively. +Normally, ReFrame sends the whole log record, which contains all of the test's variables prefixed with ``check_``. + +Note that in this example, we also set :attr:`~config.logging.handlers_perflog..httpjson..debug` to :obj:`True` so that ReFrame will simply dump the JSON record and will not attempt to send it. +Also, the :attr:`~config.logging.handlers_perflog..httpjson..json_formatter` Is optional and we will cover it in the next section. + +Let's rerun our STREAM benchmark: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + reframe --prefix=/scratch/rfm-stage/ -C config/cluster_perflogs_httpjson.py -c stream/stream_fixtures.py -r + +Notice that that there is one JSON file produced per test run named as ``httpjson_record_.json``. +You can inspect it to see the exact JSON record that ReFrame would send to the HTTP endpoint: + +.. code-block:: bash + :caption: Run with the Docker compose setup. + + jq . httpjson_record_.json + + +Customizing the JSON record +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It might be the case that the remote endpoint imposes a schema on the incoming JSON blob. +In this case, ReFrame's record will most likely be rejected. +However, you can directly format the log record to be sent to the server by setting the :attr:`~config.logging.handlers_perflog..httpjson..json_formatter` configuration option to a callable that will generate the JSON payload from ReFrame's log record. +In the following example, we remove the ``check_`` prefix from the test's attributes. + +.. literalinclude:: ../examples/tutorial/config/cluster_perflogs_httpjson.py + :caption: + :lines: 10-22 + +The format function takes the raw log record, the extras and the keys to ignore as specified in the handler configuration and returns a JSON string. +Since we can't know the exact log record attributes, we iterate over its :attr:`__dict__` items and format the record keys as we go. +Also note that we ignore all private field of the record starting with ``_``. +Rerunning the previous example with ``CUSTOM_JSON=1`` will generated the modified JSON record. diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst deleted file mode 100644 index cf35dfb01b..0000000000 --- a/docs/tutorial_advanced.rst +++ /dev/null @@ -1,981 +0,0 @@ -================================================= -Tutorial 2: Customizing Further a Regression Test -================================================= - -In this tutorial we will present common patterns that can come up when writing regression tests with ReFrame. -All examples use either the configuration files presented in :doc:`tutorial_basics`, which you can find in ``tutorials/config/tresa.py`` and ``tutorials/config/daint.py`` or build on top of them. -We also assume that the reader is already familiar with the concepts presented in the basic tutorial. -Finally, to avoid specifying the tutorial configuration every time, make sure to export it here: - -.. code:: bash - - export RFM_CONFIG_FILES=$(pwd)/tutorials/config/tresa.py:$(pwd)/tutorials/config/daint.py - - - -Parameterizing a Regression Test --------------------------------- - -We have briefly looked into parameterized tests in :doc:`tutorial_basics` where we parameterized the "Hello, World!" test based on the programming language. -Test parameterization in ReFrame is quite powerful since it allows you to create a multitude of similar tests automatically. -In this example, we will parameterize the last version of the STREAM test from the :doc:`tutorial_basics` by changing the array size, so as to check the bandwidth of the different cache levels. -Here is the adapted code with the relevant parts highlighted (for simplicity, we are interested only in the "Triad" benchmark): - -.. code-block:: console - - cat tutorials/advanced/parameterized/stream.py - - -.. literalinclude:: ../tutorials/advanced/parameterized/stream.py - :start-at: import reframe - :emphasize-lines: 7-9,44-51,55-56 - -Any ordinary ReFrame test becomes a parameterized one if the user defines parameters inside the class body of the test. -This is done using the :py:func:`~reframe.core.pipeline.RegressionTest.parameter` ReFrame built-in function, which accepts the list of parameter values. -For each parameter value ReFrame will instantiate a different regression test by assigning the corresponding value to an attribute named after the parameter. -So in this example, ReFrame will generate automatically 11 tests with different values for their :attr:`num_bytes` attribute. -From this point on, you can adapt the test based on the parameter values, as we do in this case, where we compute the STREAM array sizes, as well as the number of iterations to be performed on each benchmark, and we also compile the code accordingly. - -Let's try listing the generated tests: - -.. code-block:: console - - ./bin/reframe -c tutorials/advanced/parameterized/stream.py -l - - -.. literalinclude:: listings/stream_params.txt - :language: console - -ReFrame generates 11 tests from the single parameterized test. -When listing parameterized tests, ReFrame adds the list of parameters after the base test name using the notation ``%=``. -Each generated test gets also a unique name. -For more details on how the test names are generated for various types of tests, please refer to :ref:`test_naming_scheme`. - -Test parameterization in ReFrame is very powerful since you can parameterize your tests on anything and you can create complex parameterization spaces. -A common pattern is to parameterize a test on the environment module that loads a software in order to test different versions of it. -For this reason, ReFrame offers the :func:`~reframe.utility.find_modules` function, which allows you to parameterize a test on the available modules for a given programming environment and partition combination. -The following example will create a test for each ``GROMACS`` module found on the software stack associated with a system partition and programming environment (toolchain): - -.. code:: python - - import reframe as rfm - import reframe.utility as util - - - @rfm.simple_test - class MyTest(rfm.RegressionTest): - module_info = parameter(util.find_modules('GROMACS')) - - @run_after('init') - def process_module_info(self): - s, e, m = self.module_info - self.valid_systems = [s] - self.valid_prog_environs = [e] - self.modules = [m] - - - -More On Building Tests ----------------------- - -We have already seen how ReFrame can compile a test with a single source file. -However, ReFrame can also build tests that use Make or a configure-Make approach. -We are going to demonstrate this through a simple C++ program that computes a dot-product of two vectors and is being compiled through a Makefile. -Additionally, we can select the type of elements for the vectors at compilation time. -Here is the C++ program: - -.. code-block:: console - - cat tutorials/advanced/makefiles/src/dotprod.cpp - - -.. literalinclude:: ../tutorials/advanced/makefiles/src/dotprod.cpp - :language: cpp - :start-at: #include - -The directory structure for this test is the following: - -.. code-block:: none - - tutorials/makefiles/ - ├── maketest.py - └── src - ├── Makefile - └── dotprod.cpp - - -Let's have a look at the test itself: - -.. code-block:: console - - cat tutorials/advanced/makefiles/maketest.py - - -.. literalinclude:: ../tutorials/advanced/makefiles/maketest.py - :pyobject: MakefileTest - -First, if you're using any build system other than ``SingleSource``, you must set the :attr:`executable` attribute of the test, because ReFrame cannot know what is the actual executable to be run. -We then set the build system to :class:`~reframe.core.buildsystems.Make` and set the preprocessor flags as we would do with the :class:`SingleSource` build system. - -Let's inspect the build script generated by ReFrame: - -.. code-block:: console - - ./bin/reframe -c tutorials/advanced/makefiles/maketest.py -r - cat output/catalina/default/clang/MakefileTest_float/rfm_MakefileTest_build.sh - -.. code-block:: bash - - #!/bin/bash - - _onerror() - { - exitcode=$? - echo "-reframe: command \`$BASH_COMMAND' failed (exit code: $exitcode)" - exit $exitcode - } - - trap _onerror ERR - - make -j 1 CC="cc" CXX="CC" FC="ftn" NVCC="nvcc" CPPFLAGS="-DELEM_TYPE=float" - - -The compiler variables (``CC``, ``CXX`` etc.) are set based on the corresponding values specified in the `configuration `__ of the current environment. -We can instruct the build system to ignore the default values from the environment by setting its :attr:`~reframe.core.buildsystems.Make.flags_from_environ` attribute to false: - -.. code-block:: python - - self.build_system.flags_from_environ = False - -In this case, ``make`` will be invoked as follows: - -.. code:: - - make -j 1 CPPFLAGS="-DELEM_TYPE=float" - -Notice that the ``-j 1`` option is always generated. -We can increase the build concurrency by setting the :attr:`~reframe.core.buildsystems.Make.max_concurrency` attribute. -Finally, we may even use a custom Makefile by setting the :attr:`~reframe.core.buildsystems.Make.makefile` attribute: - -.. code-block:: python - - self.build_system.max_concurrency = 4 - self.build_system.makefile = 'Makefile_custom' - - -As a final note, as with the :class:`SingleSource` build system, it wouldn't have been necessary to specify one in this test, if we wouldn't have to set the CPPFLAGS. -ReFrame could automatically figure out the correct build system if :attr:`~reframe.core.pipeline.RegressionTest.sourcepath` refers to a directory. -ReFrame will inspect the directory and it will first try to determine whether this is a CMake or Autotools-based project. - -More details on ReFrame's build systems can be found `here `__. - - -Retrieving the source code from a Git repository -================================================ - -It might be the case that a regression test needs to clone its source code from a remote repository. -This can be achieved in two ways with ReFrame. -One way is to set the :attr:`sourcesdir` attribute to :class:`None` and explicitly clone a repository using the :attr:`~reframe.core.pipeline.RegressionTest.prebuild_cmds`: - -.. code-block:: python - - self.sourcesdir = None - self.prebuild_cmds = ['git clone https://github.com/me/myrepo .'] - -Alternatively, we can retrieve specifically a Git repository by assigning its URL directly to the :attr:`sourcesdir` attribute: - -.. code-block:: python - - self.sourcesdir = 'https://github.com/me/myrepo' - -ReFrame will attempt to clone this repository inside the stage directory by executing ``git clone .`` and will then proceed with the build procedure as usual. - -.. note:: - ReFrame recognizes only URLs in the :attr:`sourcesdir` attribute and requires passwordless access to the repository. - This means that the SCP-style repository specification will not be accepted. - You will have to specify it as URL using the ``ssh://`` protocol (see `Git documentation page `__). - - -Adding a configuration step before compiling the code -===================================================== - -It is often the case that a configuration step is needed before compiling a code with ``make``. -To address this kind of projects, ReFrame aims to offer specific abstractions for "configure-make" style of build systems. -It supports `CMake-based `__ projects through the :class:`~reframe.core.buildsystems.CMake` build system, as well as `Autotools-based `__ projects through the :class:`~reframe.core.buildsystems.Autotools` build system. - -For other build systems, you can achieve the same effect using the :class:`~reframe.core.buildsystems.Make` build system and the :attr:`~reframe.core.pipeline.RegressionTest.prebuild_cmds` for performing the configuration step. -The following code snippet will configure a code with ``./custom_configure`` before invoking ``make``: - -.. code-block:: python - - self.prebuild_cmds = ['./custom_configure -with-mylib'] - self.build_system = 'Make' - self.build_system.cppflags = ['-DHAVE_FOO'] - self.build_system.flags_from_environ = False - -The generated build script will then have the following lines: - -.. code-block:: bash - - ./custom_configure -with-mylib - make -j 1 CPPFLAGS='-DHAVE_FOO' - - -Writing a Run-Only Regression Test ----------------------------------- - -There are cases when it is desirable to perform regression testing for an already built executable. -In the following test we use simply the ``echo`` Bash shell command to print a random integer between specific lower and upper bounds. -Here is the full regression test: - -.. code-block:: console - - cat tutorials/advanced/runonly/echorand.py - - -.. literalinclude:: ../tutorials/advanced/runonly/echorand.py - :start-at: import reframe - :emphasize-lines: 6 - -There is nothing special for this test compared to those presented so far except that it derives from the :class:`~reframe.core.pipeline.RunOnlyRegressionTest` class. -Note that setting the :attr:`~reframe.core.pipeline.RegressionTest.executable` in this type of test is always required. -Run-only regression tests may also have resources, as for instance a pre-compiled executable or some input data. -These resources may reside under the ``src/`` directory or under any directory specified in the :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir` attribute. -These resources will be copied to the stage directory at the beginning of the run phase. - - -Writing a Compile-Only Regression Test --------------------------------------- - -ReFrame provides the option to write compile-only tests which consist only of a compilation phase without a specified executable. -This kind of tests must derive from the :class:`~reframe.core.pipeline.CompileOnlyRegressionTest` class provided by the framework. -The following test is a compile-only version of the :class:`MakefileTest` presented `previously <#more-on-building-tests>`__ which checks that no warnings are issued by the compiler: - -.. code-block:: console - - cat tutorials/advanced/makefiles/maketest.py - - -.. literalinclude:: ../tutorials/advanced/makefiles/maketest.py - :pyobject: MakeOnlyTest - :emphasize-lines: 2 - -What is worth noting here is that the standard output and standard error of the test, which are accessible through the :attr:`~reframe.core.pipeline.RegressionTest.stdout` and :attr:`~reframe.core.pipeline.RegressionTest.stderr` attributes, correspond now to the standard output and error of the compilation command. -Therefore sanity checking can be done in exactly the same way as with a normal test. - - -Grouping parameter packs ------------------------- - -.. versionadded:: 3.4.2 - - -In the dot product example shown above, we had two independent tests that defined the same :attr:`elem_type` parameter. -And the two tests cannot have a parent-child relationship, since one of them is a run-only test and the other is a compile-only one. -ReFrame offers the :class:`~reframe.core.pipeline.RegressionMixin` class that allows you to group parameters and other `builtins `__ that are meant to be reused over otherwise unrelated tests. -In the example below, we create an :class:`ElemTypeParam` mixin that holds the definition of the :attr:`elem_type` parameter which is inherited by both the concrete test classes: - -.. literalinclude:: ../tutorials/advanced/makefiles/maketest_mixin.py - :start-at: import reframe - :emphasize-lines: 5-6,10,30 - - -Notice how the parameters are expanded in each of the individual tests: - -.. code-block:: console - - ./bin/reframe -c tutorials/advanced/makefiles/maketest_mixin.py -l - -.. literalinclude:: listings/maketest_mixin.txt - :language: console - - - -Applying a Sanity Function Iteratively --------------------------------------- - -It is often the case that a common sanity function has to be applied many times. -The following script prints 100 random integers between the limits given by the environment variables ``LOWER`` and ``UPPER``. - -.. code-block:: console - - cat tutorials/advanced/random/src/random_numbers.sh - - -.. literalinclude:: ../tutorials/advanced/random/src/random_numbers.sh - :language: bash - :start-after: # rfmdocstart: random_numbers - :end-before: # rfmdocend: random_numbers - -In the corresponding regression test we want to check that all the random numbers generated lie between the two limits, which means that a common sanity check has to be applied to all the printed random numbers. -Here is the corresponding regression test: - -.. code-block:: console - - cat tutorials/advanced/random/randint.py - - -.. literalinclude:: ../tutorials/advanced/random/randint.py - :start-at: import reframe - :emphasize-lines: 12- - -First, we extract all the generated random numbers from the output. -What we want to do is to apply iteratively the :func:`~reframe.utility.sanity.assert_bounded` sanity function for each number. -The problem here is that we cannot simply iterate over the ``numbers`` list, because that would trigger prematurely the evaluation of the :func:`~reframe.utility.sanity.extractall`. -We want to defer also the iteration. -This can be achieved by using the :func:`~reframe.utility.sanity.map` ReFrame sanity function, which is a replacement of Python's built-in :py:func:`map` function and does exactly what we want: it applies a function on all the elements of an iterable and returns another iterable with the transformed elements. -Passing the result of the :py:func:`map` function to the :func:`~reframe.utility.sanity.all` sanity function ensures that all the elements lie between the desired bounds. - -There is still a small complication that needs to be addressed. -As a direct replacement of the built-in :py:func:`all` function, ReFrame's :func:`~reframe.utility.sanity.all` sanity function returns :class:`True` for empty iterables, which is not what we want. -So we must make sure that all 100 numbers are generated. -This is achieved by the ``sn.assert_eq(sn.count(numbers), 100)`` statement, which uses the :func:`~reframe.utility.sanity.count` sanity function for counting the generated numbers. -Finally, we need to combine these two conditions to a single deferred expression that will be returned by the test's :attr:`@sanity_function`. -We accomplish this by using the :func:`~reframe.utility.sanity.all` sanity function. - -For more information about how exactly sanity functions work and how their execution is deferred, please refer to :doc:`deferrables`. - -.. note:: - .. versionadded:: 2.13 - ReFrame offers also the :func:`~reframe.utility.sanity.allx` sanity function which, conversely to the builtin :func:`all()` function, will return :class:`False` if its iterable argument is empty. - - -Customizing the Test Job Script -------------------------------- - -It is often the case that we need to run some commands before or after the parallel launch of our executable. -This can be easily achieved by using the :attr:`~reframe.core.pipeline.RegressionTest.prerun_cmds` and :attr:`~reframe.core.pipeline.RegressionTest.postrun_cmds` attributes of a ReFrame test. - -The following example is a slightly modified version of the random numbers test presented `above <#applying-a-sanity-function-iteratively>`__. -The lower and upper limits for the random numbers are now set inside a helper shell script in ``limits.sh`` located in the test's resources, which we need to source before running our tests. -Additionally, we want also to print ``FINISHED`` after our executable has finished. -Here is the modified test file: - -.. code-block:: console - - cat tutorials/advanced/random/prepostrun.py - - -.. literalinclude:: ../tutorials/advanced/random/prepostrun.py - :start-at: import reframe - :emphasize-lines: 10-11,19,22 - -The :attr:`prerun_cmds` and :attr:`postrun_cmds` are lists of commands to be emitted in the generated job script before and after the parallel launch of the executable. -Obviously, the working directory for these commands is that of the job script itself, which is the stage directory of the test. -The generated job script for this test looks like the following: - -.. code-block:: console - - ./bin/reframe -c tutorials/advanced/random/prepostrun.py -r - cat output/catalina/default/gnu/PrepostRunTest/rfm_PrepostRunTest_job.sh - -.. code-block:: bash - - #!/bin/bash - source limits.sh - ./random_numbers.sh - echo FINISHED - -Generally, ReFrame generates the job shell scripts using the following pattern: - -.. code-block:: bash - - #!/bin/bash -l - {job_scheduler_preamble} - {prepare_cmds} - {env_load_cmds} - {prerun_cmds} - {parallel_launcher} {executable} {executable_opts} - {postrun_cmds} - -The ``job_scheduler_preamble`` contains the backend job scheduler directives that control the job allocation. -The ``prepare_cmds`` are commands that can be emitted before the test environment commands. -These can be specified with the :attr:`~config.systems.partitions.prepare_cmds` partition configuration option. -The ``env_load_cmds`` are the necessary commands for setting up the environment of the test. -These include any modules or environment variables set at the `system partition level `__ or any `modules `__ or `environment variables `__ set at the test level. -Then the commands specified in :attr:`~reframe.core.pipeline.RegressionTest.prerun_cmds` follow, while those specified in the :attr:`~reframe.core.pipeline.RegressionTest.postrun_cmds` come after the launch of the parallel job. -The parallel launch itself consists of three parts: - -#. The parallel launcher program (e.g., ``srun``, ``mpirun`` etc.) with its options, -#. the regression test executable as specified in the :attr:`~reframe.core.pipeline.RegressionTest.executable` attribute and -#. the options to be passed to the executable as specified in the :attr:`~reframe.core.pipeline.RegressionTest.executable_opts` attribute. - - -Adding job scheduler options per test -===================================== - -Sometimes a test needs to pass additional job scheduler options to the automatically generated job script. -This is fairly easy to achieve with ReFrame. -In the following test we want to test whether the ``--mem`` option of Slurm works as expected. -We compiled and ran a program that consumes all the available memory of the node, but we want to restrict the available memory with the ``--mem`` option. -Here is the test: - -.. code-block:: console - - cat tutorials/advanced/jobopts/eatmemory.py - - -.. literalinclude:: ../tutorials/advanced/jobopts/eatmemory.py - :pyobject: MemoryLimitTest - :emphasize-lines: 12-14 - -Each ReFrame test has an associated `run job descriptor `__ which represents the scheduler job that will be used to run this test. -This object has an :attr:`options` attribute, which can be used to pass arbitrary options to the scheduler. -The job descriptor is initialized by the framework during the `setup `__ pipeline phase. -For this reason, we cannot directly set the job options inside the test constructor and we have to use a pipeline hook that runs before running (i.e., submitting the test). - -Let's run the test and inspect the generated job script: - -.. code-block:: console - - ./bin/reframe -c tutorials/advanced/jobopts/eatmemory.py -n MemoryLimitTest -r - cat output/daint/gpu/gnu/MemoryLimitTest/rfm_MemoryLimitTest_job.sh - -.. code:: bash - - #!/bin/bash - #SBATCH --job-name="rfm_MemoryLimitTest_job" - #SBATCH --ntasks=1 - #SBATCH --output=rfm_MemoryLimitTest_job.out - #SBATCH --error=rfm_MemoryLimitTest_job.err - #SBATCH --time=0:10:0 - #SBATCH -A csstaff - #SBATCH --constraint=gpu - #SBATCH --mem=1000 - module unload PrgEnv-cray - module load PrgEnv-gnu - srun ./MemoryLimitTest 2000M - - -The job options specified inside a ReFrame test are always the last to be emitted in the job script preamble and do not affect the options that are passed implicitly through other test attributes or configuration options. - -There is a small problem with this test though. -What if we change the job scheduler in that partition or what if we want to port the test to a different system that does not use Slurm and another option is needed to achieve the same result. -The obvious answer is to adapt the test, but is there a more portable way? -The answer is yes and this can be achieved through so-called *extra resources*. -ReFrame gives you the possibility to associate scheduler options to a "resource" managed by the partition scheduler. -You can then use those resources transparently from within your test. - -To achieve this in our case, we first need to define a ``memory`` resource in the configuration of both of our ``daint`` partitions: - - -.. literalinclude:: ../tutorials/config/daint_mem.py - :start-after: # rfmdocstart: memory - :end-before: # rfmdocend: memory - -Notice that we do not define the resource for all the partitions, but only for those that it makes sense. -Each resource has a name and a set of scheduler options that will be passed to the scheduler when this resource will be requested by the test. -The options specification can contain placeholders, whose value will also be set from the test. -Let's see how we can rewrite the :class:`MemoryLimitTest` using the ``memory`` resource instead of passing the ``--mem`` scheduler option explicitly. - -.. code-block:: console - - cat tutorials/advanced/jobopts/eatmemory.py - - -.. literalinclude:: ../tutorials/advanced/jobopts/eatmemory.py - :pyobject: MemoryLimitWithResourcesTest - :emphasize-lines: 7-9 - -The extra resources that the test needs to obtain through its scheduler are specified in the :attr:`~reframe.core.pipeline.RegressionTest.extra_resources` attribute, which is a dictionary with the resource names as its keys and another dictionary assigning values to the resource placeholders as its values. -As you can see, this syntax is completely scheduler-agnostic. -If the requested resource is not defined for the current partition, it will be simply ignored. - -You can now run and verify that the generated job script contains the ``--mem`` option: - -.. code-block:: none - - ./bin/reframe -c tutorials/advanced/jobopts/eatmemory.py -n MemoryLimitWithResourcesTest -r - cat output/daint/gpu/gnu/MemoryLimitWithResourcesTest/rfm_MemoryLimitWithResourcesTest_job.sh - - -Modifying the parallel launcher command -======================================= - -Another relatively common need is to modify the parallel launcher command. -ReFrame gives the ability to do that and we will see some examples in this section. - -The most common case is to pass arguments to the launcher command that you cannot normally pass as job options. -The ``--cpu-bind`` of ``srun`` is such an example. -Inside a ReFrame test, you can access the parallel launcher through the :attr:`~reframe.core.schedulers.Job.launcher` of the job descriptor. -This object handles all the details of how the parallel launch command will be emitted. -In the following test we run a CPU affinity test using `this `__ utility and we will pin the threads using the ``--cpu-bind`` option: - -.. code-block:: console - - cat tutorials/advanced/affinity/affinity.py - - -.. literalinclude:: ../tutorials/advanced/affinity/affinity.py - :start-at: import reframe - -The approach is identical to the approach we took in the :class:`MemoryLimitTest` test `above <#adding-job-scheduler-options-per-test>`__, except that we now set the launcher options. - -.. note:: - - The sanity checking in a real affinity checking test would be much more complex than this. - -Another scenario that might often arise when testing parallel debuggers is the need to wrap the launcher command with the debugger command. -For example, in order to debug a parallel program with `ARM DDT `__, you would need to invoke the program like this: ``ddt [OPTIONS] srun [OPTIONS]``. -ReFrame allows you to wrap the launcher command without the test needing to know which is the actual parallel launcher command for the current partition. -This can be achieved with the following pipeline hook: - - -.. code:: python - - import reframe as rfm - from reframe.core.launchers import LauncherWrapper - - class DebuggerTest(rfm.RunOnlyRegressionTest): - ... - - @run_before('run') - def set_launcher(self): - self.job.launcher = LauncherWrapper(self.job.launcher, 'ddt', - ['--offline']) - -The :class:`~reframe.core.launchers.LauncherWrapper` is a pseudo-launcher that wraps another one and allows you to prepend anything to it. -In this case the resulting parallel launch command, if the current partition uses native Slurm, will be ``ddt --offline srun [OPTIONS]``. - - -Replacing the parallel launcher -=============================== - -Sometimes you might need to replace completely the partition's launcher command, because the software you are testing might use its own parallel launcher. -Examples are `ipyparallel `__, the `GREASY `__ high-throughput scheduler, as well as some visualization software. -The trick here is to replace the parallel launcher with the local one, which practically does not emit any launch command, and by now you should almost be able to do it all by yourself: - -.. code:: python - - import reframe as rfm - from reframe.core.backends import getlauncher - - - class CustomLauncherTest(rfm.RunOnlyRegressionTest): - ... - executable = 'custom_scheduler' - executable_opts = [...] - - @run_before('run') - def replace_launcher(self): - self.job.launcher = getlauncher('local')() - - -The :func:`~reframe.core.backends.getlauncher` function takes the `registered `__ name of a launcher and returns the class that implements it. -You then instantiate the launcher and assign to the :attr:`~reframe.core.schedulers.Job.launcher` attribute of the job descriptor. - - -Adding more parallel launch commands -==================================== - -ReFrame uses a parallel launcher by default for anything defined explicitly or implicitly in the :attr:`~reframe.core.pipeline.RegressionTest.executable` test attribute. -But what if we want to generate multiple parallel launch commands? -One straightforward solution is to hardcode the parallel launch command inside the :attr:`~reframe.core.pipeline.RegressionTest.prerun_cmds` or :attr:`~reframe.core.pipeline.RegressionTest.postrun_cmds`, but this is not so portable. -The best way is to ask ReFrame to emit the parallel launch command for you. -The following is a simple test for demonstration purposes that runs the ``hostname`` command several times using a parallel launcher. -It resembles a scaling test, except that all happens inside a single ReFrame test, instead of launching multiple instances of a parameterized test. - -.. code-block:: console - - cat tutorials/advanced/multilaunch/multilaunch.py - - -.. literalinclude:: ../tutorials/advanced/multilaunch/multilaunch.py - :start-at: import reframe - :emphasize-lines: 13-19 - -The additional parallel launch commands are inserted in either the :attr:`prerun_cmds` or :attr:`postrun_cmds` lists. -To retrieve the actual parallel launch command for the current partition that the test is running on, you can use the :func:`~reframe.core.launchers.Launcher.run_command` method of the launcher object. -Let's see how the generated job script looks like: - -.. code-block:: none - - ./bin/reframe -c tutorials/advanced/multilaunch/multilaunch.py -r - cat output/daint/gpu/builtin/MultiLaunchTest/rfm_MultiLaunchTest_job.sh - -.. code:: bash - - #!/bin/bash - #SBATCH --job-name="rfm_MultiLaunchTest_job" - #SBATCH --ntasks=4 - #SBATCH --ntasks-per-node=1 - #SBATCH --output=rfm_MultiLaunchTest_job.out - #SBATCH --error=rfm_MultiLaunchTest_job.err - #SBATCH --time=0:10:0 - #SBATCH -A csstaff - #SBATCH --constraint=gpu - srun -n 1 hostname - srun -n 2 hostname - srun -n 3 hostname - srun hostname - - -The first three ``srun`` commands are emitted through the :attr:`prerun_cmds` whereas the last one comes from the test's :attr:`executable` attribute. - - -.. _custom_launchers: - -Adding a custom launcher to a partition -======================================= - -.. versionadded:: 4.0.0 - -An alternative to the approaches above would be to define your own custom parallel launcher and register it with the framework. -You could then use it as the launcher of a system partition in the configuration and use it in multiple tests. - -Each `launcher `__ needs to implement the :func:`~reframe.core.launchers.JobLauncher.command` method and can optionally change the default :func:`~reframe.core.launchers.JobLauncher.run_command` method. - -As an example of how easy it is to define a new parallel launcher backend, here is the actual implementation of the ``mpirun`` launcher: - -.. code:: python - - from reframe.core.backends import register_launcher - from reframe.core.launchers import JobLauncher - - - @register_launcher('mpirun') - class MpirunLauncher(JobLauncher): - def command(self, job): - return ['mpirun', '-np', str(job.num_tasks)] - - -The :func:`~reframe.core.launchers.JobLauncher.command` returns a list of command tokens that will be combined with any user-supplied `options `__ by the :func:`~reframe.core.launchers.JobLauncher.run_command` method to generate the actual launcher command line. -Notice you can use the ``job`` argument to get job-specific information that will allow you to construct the correct launcher invocation. - -If you use a Python-based configuration file, you can define your custom launcher directly inside your config as follows: - -.. code:: python - - from reframe.core.backends import register_launcher - from reframe.core.launchers import JobLauncher - - - @register_launcher('slrun') - class MySmartLauncher(JobLauncher): - def command(self, job): - return ['slrun', ...] - - site_configuration = { - 'systems': [ - { - 'name': 'my_system', - 'partitions': [ - { - 'name': 'my_partition', - 'launcher': 'slrun' - ... - } - ], - ... - }, - ... - ], - ... - } - - -Flexible Regression Tests -------------------------- - -.. versionadded:: 2.15 - -ReFrame can automatically set the number of tasks of a particular test, if its :attr:`~reframe.core.pipeline.RegressionTest.num_tasks` attribute is set to a negative value or zero. -In ReFrame's terminology, such tests are called *flexible*. -Negative values indicate the minimum number of tasks that are acceptable for this test (a value of ``-4`` indicates that at least ``4`` tasks are required). -A zero value indicates the default minimum number of tasks which is equal to :attr:`~reframe.core.pipeline.RegressionTest.num_tasks_per_node`. - -By default, ReFrame will spawn such a test on all the idle nodes of the current system partition, but this behavior can be adjusted with the |--flex-alloc-nodes|_ command-line option. -Flexible tests are very useful for diagnostics tests, e.g., tests for checking the health of a whole set nodes. -In this example, we demonstrate this feature through a simple test that runs ``hostname``. -The test will verify that all the nodes print the expected host name: - -.. code-block:: console - - cat tutorials/advanced/flexnodes/flextest.py - - -.. literalinclude:: ../tutorials/advanced/flexnodes/flextest.py - :start-at: import reframe - :emphasize-lines: 10- - -The first thing to notice in this test is that :attr:`~reframe.core.pipeline.RegressionTest.num_tasks` is set to zero as default, which is a requirement for flexible tests. -However, with flexible tests, this value is updated right after the job completes to the actual number of tasks that were used. -Consequently, this allows the sanity function of the test to assert that the number host names printed matches :attr:`~reframe.core.pipeline.RegressionTest.num_tasks`. - -.. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` -.. _--flex-alloc-nodes: manpage.html#cmdoption-flex-alloc-nodes - - -.. tip:: - - If you want to run multiple flexible tests at once, it's better to run them using the serial execution policy, because the first test might take all the available nodes and will cause the rest to fail immediately, since there will be no available nodes for them. - - -Testing containerized applications ----------------------------------- - -.. versionadded:: 2.20 - - -ReFrame can be used also to test applications that run inside a container. -First, we need to enable the container platform support in ReFrame's configuration and, specifically, at the partition configuration level: - -.. literalinclude:: ../tutorials/config/daint_containers.py - :start-after: # rfmdocstart: containers - :end-before: # rfmdocend: containers - :emphasize-lines: 9-18 - -For each partition, users can define a list of all supported container platforms using the :attr:`~config.systems.partitions.container_platforms` configuration parameter. -In this case, we define the `Sarus `__ platform for which we set the :attr:`~config.systems.partitions.container_platforms.modules` parameter in order to instruct ReFrame to load the ``sarus`` module, whenever it needs to run with this container platform. -Similarly, we add an entry for the `Singularity `__ platform. -Optionally, users are allowed to set the ``default`` attribute to :obj:`True` in order to mark a specific container platform as the default of that partition (see below on how this information is being used). -If no default container platform is specified explicitly, then always the first in the list will be considered as successful. - - -The following parameterized test, will create two tests, one for each of the supported container platforms: - -.. code-block:: console - - cat tutorials/advanced/containers/container_test.py - - -.. literalinclude:: ../tutorials/advanced/containers/container_test.py - :start-at: import reframe - :emphasize-lines: 11-19 - -A container-based test can be written as :class:`~reframe.core.pipeline.RunOnlyRegressionTest` that sets the :attr:`~reframe.core.pipeline.RegressionTest.container_platform` attribute. -This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. -It is not necessary to specify this attribute, in which case, the default container platform of the current partition will be used. -You can still differentiate your test based on the actual container platform that is being used by checking the ``self.container_platform.name`` variable. - -As soon as the container platform to be used is determined, you need to specify the container image to use by setting the :attr:`~reframe.core.containers.ContainerPlatform.image`. -If the image is not specified, then the container logic is skipped and the test executes as if the :attr:`~reframe.core.pipeline.RegressionTest.container_platform` was never set. - -In the ``Singularity`` test variant, we add the ``docker://`` prefix to the image name, in order to instruct ``Singularity`` to pull the image from `DockerHub `__. -The default command that the container runs can be overwritten by setting the :attr:`~reframe.core.containers.ContainerPlatform.command` attribute of the container platform. - -The :attr:`~reframe.core.containers.ContainerPlatform.image` is the only mandatory attribute for container-based checks. -It is important to note that the :attr:`~reframe.core.pipeline.RegressionTest.executable` and :attr:`~reframe.core.pipeline.RegressionTest.executable_opts` attributes of the actual test are ignored if the containerized code path is taken, i.e., when :attr:`~reframe.core.containers.ContainerPlatform.image` is not :obj:`None`. - -ReFrame will run the container according to the given platform as follows: - -.. code-block:: bash - - # Sarus - sarus run --mount=type=bind,source="/path/to/test/stagedir",destination="/rfm_workdir" ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' - - # Singularity - singularity exec -B"/path/to/test/stagedir:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' - - -In the ``Sarus`` case, ReFrame will prepend the following command in order to pull the container image before running the container: - -.. code-block:: bash - - sarus pull ubuntu:18.04 - - -This is the default behavior of ReFrame, which can be changed if pulling the image is not desired by setting the :attr:`~reframe.core.containers.ContainerPlatform.pull_image` attribute to :class:`False`. -By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container. -Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. -Besides the stage directory, additional mount points can be specified through the :attr:`~reframe.core.pipeline.RegressionTest.container_platform.mount_points` attribute: - -.. code-block:: python - - self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), - ('/path/to/host/dir2', '/path/to/container/mount_point2')] - - -The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy artifacts as needed. -These artifacts will therefore be available inside the stage directory after the container execution finishes. -This is very useful if the artifacts are needed for the sanity or performance checks. -If the copy is not performed by the default container command, the user can override this command by settings the :attr:`~reframe.core.containers.ContainerPlatform.command` attribute such as to include the appropriate copy commands. -In the current test, the output of the ``cat /etc/os-release`` is available both in the standard output as well as in the ``release.txt`` file, since we have used the command: - -.. code-block:: bash - - bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' - - -and ``/rfm_workdir`` corresponds to the stage directory on the host system. -Therefore, the ``release.txt`` file can now be used in the subsequent sanity checks: - -.. literalinclude:: ../tutorials/advanced/containers/container_test.py - :start-at: @sanity_function - :end-at: return - - -For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. -On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`. - - -.. versionchanged:: 3.12.0 - There is no need any more to explicitly set the :attr:`container_platform` in the test. - This is automatically initialized from the default platform of the current partition. - - - -Combining containerized and native application tests -==================================================== - -.. versionadded:: 3.12.0 - -It is very easy in ReFrame to have a single run-only test to either test the native or the containerized version of an application. -This is possible, since the framework will only take the "containerized" code path only if the :attr:`~reframe.core.containers.ContainerPlatform.image` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.container_platform` is not :obj:`None`. -Otherwise, the *bare metal* version of the tested application will be run. -The following test uses exactly this trick to test a series of GROMACS images as well as the native one provided on the Piz Daint supercomputer. -It also extends the GROMACS benchmark tests that are provided with ReFrame's test library (see :doc:`hpctestlib`). -For simplicity, we are assuming a single system here (the hybrid partition of Piz Daint) and we set fixed values for the :attr:`num_cpus_per_task` as well as the ``-ntomp`` option of GROMACS (NB: in a real-world test we would use the auto-detected processor topology information to set these values; see :ref:`proc-autodetection` for more information). -We also redefine and restrict the benchmark's parameters ``benchmark_info`` and ``nb_impl`` to the values that are of interest for the demonstration of this test. -Finally, we also reset the executable to use ``gmx`` instead of the ``gmx_mpi`` that is used from the library test. - - -.. literalinclude:: ../tutorials/advanced/containers/gromacs_test.py - :start-at: import reframe - -All this test does in addition to the library test it inherits from is to set the :attr:`~reframe.core.containers.ContainerPlatform.image` and the :attr:`~reframe.core.containers.ContainerPlatform.command` attributes of the :attr:`~reframe.core.pipeline.RegressionTest.container_platform`. -The former is set from the ``gromacs_image`` test parameter whereas the latter from the test's :attr:`~reframe.core.pipeline.RegressionTest.executable` and :attr:`~reframe.core.pipeline.RegressionTest.executable_opts` attributes. -Remember that these attributes are ignored if the framework takes the path of launching a container. -Finally, if the image is :obj:`None` we handle the case of the native run, in which case we load the modules required to run GROMACS natively on the target system. - -In the following, we run the GPU version of a single benchmark with a series of images from NVIDIA and natively: - -.. code-block:: console - - $ ./bin/reframe -C tutorials/config/daint.py -c tutorials/advanced/containers/gromacs_test.py -r - -.. code-block:: console - - [==========] Running 6 check(s) - [==========] Started on Fri Jun 17 16:20:16 2022 - - [----------] start processing checks - [ RUN ] gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2022.1 @daint:gpu+gnu - [ RUN ] gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2021.3 @daint:gpu+gnu - [ RUN ] gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2021 @daint:gpu+gnu - [ RUN ] gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2020.2 @daint:gpu+gnu - [ RUN ] gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2020 @daint:gpu+gnu - [ RUN ] gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=None @daint:gpu+gnu - [ OK ] (1/6) gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2020.2 @daint:gpu+gnu - [ OK ] (2/6) gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2020 @daint:gpu+gnu - [ OK ] (3/6) gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=None @daint:gpu+gnu - [ OK ] (4/6) gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2022.1 @daint:gpu+gnu - [ OK ] (5/6) gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2021 @daint:gpu+gnu - [ OK ] (6/6) gromacs_containerized_test %benchmark_info=HECBioSim/hEGFRDimerSmallerPL %nb_impl=gpu %gromacs_image=nvcr.io/hpc/gromacs:2021.3 @daint:gpu+gnu - [----------] all spawned checks have finished - - [ PASSED ] Ran 6/6 test case(s) from 6 check(s) (0 failure(s), 0 skipped) - [==========] Finished on Fri Jun 17 16:23:47 2022 - - -We can also inspect the generated job scripts for the native and a containerized run: - -.. code-block:: console - - cat output/daint/gpu/gnu/gromacs_containerized_test_0/rfm_gromacs_containerized_test_0_job.sh - -.. code-block:: bash - - #!/bin/bash - #SBATCH --job-name="rfm_gromacs_containerized_test_0_job" - #SBATCH --ntasks=1 - #SBATCH --ntasks-per-node=1 - #SBATCH --cpus-per-task=12 - #SBATCH --output=rfm_gromacs_containerized_test_0_job.out - #SBATCH --error=rfm_gromacs_containerized_test_0_job.err - #SBATCH -A csstaff - #SBATCH --constraint=gpu - #SBATCH --hint=nomultithread - module unload PrgEnv-cray - module load PrgEnv-gnu - module load daint-gpu - module load GROMACS - curl -LJO https://github.com/victorusu/GROMACS_Benchmark_Suite/raw/1.0.0/HECBioSim/hEGFRDimerSmallerPL/benchmark.tpr - srun gmx mdrun -dlb yes -ntomp 12 -npme -1 -v -nb gpu -s benchmark.tpr - -And the containerized run: - -.. code-block:: console - - cat output/daint/gpu/gnu/gromacs_containerized_test_1/rfm_gromacs_containerized_test_1_job.sh - -.. code-block:: bash - - #!/bin/bash - #SBATCH --job-name="rfm_gromacs_containerized_test_1_job" - #SBATCH --ntasks=1 - #SBATCH --ntasks-per-node=1 - #SBATCH --cpus-per-task=12 - #SBATCH --output=rfm_gromacs_containerized_test_1_job.out - #SBATCH --error=rfm_gromacs_containerized_test_1_job.err - #SBATCH -A csstaff - #SBATCH --constraint=gpu - #SBATCH --hint=nomultithread - module unload PrgEnv-cray - module load PrgEnv-gnu - module load sarus - curl -LJO https://github.com/victorusu/GROMACS_Benchmark_Suite/raw/1.0.0/HECBioSim/hEGFRDimerSmallerPL/benchmark.tpr - sarus pull nvcr.io/hpc/gromacs:2020 - srun sarus run --mount=type=bind,source="/users/user/Devel/reframe/stage/daint/gpu/gnu/gromacs_containerized_test_43",destination="/rfm_workdir" -w /rfm_workdir nvcr.io/hpc/gromacs:2020 gmx mdrun -dlb yes -ntomp 12 -npme -1 -v -nb gpu -s benchmark.tpr - - - -Writing reusable tests ----------------------- - -.. versionadded:: 3.5.0 - -So far, all the examples shown above were tight to a particular system or configuration, which makes reusing these tests in other systems not straightforward. -However, the introduction of the :py:func:`~reframe.core.pipeline.RegressionMixin.parameter` and :py:func:`~reframe.core.pipeline.RegressionMixin.variable` ReFrame built-ins solves this problem, eliminating the need to specify any of the test variables in the :func:`__init__` method and simplifying code reuse. -Hence, readers who are not familiar with these built-in functions are encouraged to read their basic use examples (see :py:func:`~reframe.core.pipeline.RegressionMixin.parameter` and :py:func:`~reframe.core.pipeline.RegressionMixin.variable`) before delving any deeper into this tutorial. - -In essence, parameters and variables can be treated as simple class attributes, which allows us to leverage Python's class inheritance and write more modular tests. -For simplicity, we illustrate this concept with the above :class:`ContainerTest` example, where the goal here is to re-write this test as a library that users can simply import from and derive their tests without having to rewrite the bulk of the test. -Also, for illustrative purposes, we parameterize this library test on a few different image tags (the above example just used ``ubuntu:18.04``) and throw the container commands into a separate bash script just to create some source files. -Thus, removing all the system and configuration specific variables, and moving as many assignments as possible into the class body, the system agnostic library test looks as follows: - -.. code-block:: console - - cat tutorials/advanced/library/lib/__init__.py - - -.. literalinclude:: ../tutorials/advanced/library/lib/__init__.py - :start-at: import reframe - :emphasize-lines: 8-17 - -Note that the class :class:`ContainerBase` is not decorated since it does not specify the required variables ``valid_systems`` and ``valid_prog_environs``, and it declares the ``platform`` parameter without any defined values assigned. -Hence, the user can simply derive from this test and specialize it to use the desired container platforms. -Since the parameters are defined directly in the class body, the user is also free to override or extend any of the other parameters in a derived test. -In this example, we have parameterized the base test to run with the ``ubuntu:18.04`` and ``ubuntu:20.04`` images, but these values from ``dist`` (and also the ``dist_name`` variable) could be modified by the derived class if needed. - -On the other hand, the rest of the test depends on the values from the test parameters, and a parameter is only assigned a specific value after the class has been instantiated. -Thus, the rest of the test is expressed as hooks, without the need to write anything in the :func:`__init__` method. -In fact, writing the test in this way permits having hooks that depend on undefined variables or parameters. -This is the case with the :func:`set_container_platform` hook, which depends on the undefined parameter ``platform``. -Hence, the derived test **must** define all the required parameters and variables; otherwise ReFrame will notice that the test is not well defined and will raise an error accordingly. - -Before moving ahead with the derived test, note that the :class:`ContainerBase` class takes the additional argument ``pin_prefix=True``, which locks the prefix of all derived tests to this base test. -This will allow the retrieval of the sources located in the library by any derived test, regardless of what their containing directory is. - -.. code-block:: console - - cat tutorials/advanced/library/lib/src/get_os_release.sh - - -.. literalinclude:: ../tutorials/advanced/library/lib/src/get_os_release.sh - -Now from the user's perspective, the only thing to do is to import the above base test and specify the required variables and parameters. -For consistency with the above example, we set the ``platform`` parameter to use Sarus and Singularity, and we configure the test to run on Piz Daint with the built-in programming environment. -Hence, the above :class:`ContainerTest` is now reduced to the following: - -.. code-block:: console - - cat tutorials/advanced/library/usr/container_test.py - - -.. literalinclude:: ../tutorials/advanced/library/usr/container_test.py - :start-after: import reframe - -In a similar fashion, any other user could reuse the above :class:`ContainerBase` class and write the test for their own system with a few lines of code. - -*Happy test sharing!* diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst deleted file mode 100644 index 00643a6025..0000000000 --- a/docs/tutorial_basics.rst +++ /dev/null @@ -1,756 +0,0 @@ -========================================== - Tutorial 1: Getting Started with ReFrame -========================================== - -.. versionadded:: 3.1 - -This tutorial will give you a first overview of ReFrame and will acquaint you with its basic concepts. -We will start with a simple "Hello, World!" test running with the default configuration and we will expand the example along the way. -We will also explore performance tests and port our tests to an HPC cluster. -The examples of this tutorial can be found under :obj:`tutorials/basics/`. - - -Getting Ready -------------- - -All you need to start off with this tutorial is to have `installed `__ ReFrame. -If you haven't done so yet, all you need is Python 3.6 and above and to follow the steps below: - - -.. code:: bash - - git clone https://github.com/reframe-hpc/reframe.git - cd reframe - ./bootstrap.sh - ./bin/reframe -V - -We're now good to go! - - -The "Hello, World!" test ------------------------- - -As simple as it may sound, a series of "naive" "Hello, World!" tests can reveal lots of regressions in the programming environment of HPC clusters, but the bare minimum of those also serves perfectly the purpose of starting this tutorial. -Here is its C version: - -.. code-block:: console - - cat tutorials/basics/hello/src/hello.c - - -.. literalinclude:: ../tutorials/basics/hello/src/hello.c - :language: c - :start-at: #include - - -And here is the ReFrame version of it: - -.. code-block:: console - - cat tutorials/basics/hello/hello1.py - - -.. literalinclude:: ../tutorials/basics/hello/hello1.py - :start-at: import reframe - - -Regression tests in ReFrame are specially decorated classes that ultimately derive from :class:`~reframe.core.pipeline.RegressionTest`. -The :func:`@simple_test ` decorator registers a test class with ReFrame and makes it available to the framework. -The test variables are essentially attributes of the test class and can be defined directly in the class body. -Each test must always set the :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs` attributes. -These define the systems and/or system partitions that this test is allowed to run on, as well as the programming environments that it is valid for. -A programming environment is essentially a compiler toolchain. -We will see later on in the tutorial how a programming environment can be defined. -The generic configuration of ReFrame assumes a single programming environment named ``builtin`` which comprises a C compiler that can be invoked with ``cc``. -In this particular test we set both these attributes to ``['*']``, essentially allowing this test to run everywhere. - -A ReFrame test must either define an executable to execute or a source file (or source code) to be compiled. -In this example, it is enough to define the source file of our hello program. -ReFrame knows the executable that was produced and will use that to run the test. - -Finally, every regression test must always decorate a member function as the test's :func:`@sanity_function`. -This decorated function is converted into a `lazily evaluated `__ expression that asserts the sanity of the test. -In this particular case, the specified sanity function checks that the executable has produced the desired phrase into the test's standard output :attr:`~reframe.core.pipeline.RegressionTest.stdout`. -Note that ReFrame does not determine the success of a test by its exit code. -Instead, the assessment of success is responsibility of the test itself. - -Before running the test let's inspect the directory structure surrounding it: - -.. code-block:: none - - tutorials/basics/hello - ├── hello1.py - └── src - └── hello.c - -Our test is ``hello1.py`` and its resources, i.e., the ``hello.c`` source file, are located inside the ``src/`` subdirectory. -If not specified otherwise, the :attr:`~reframe.core.pipeline.RegressionTest.sourcepath` attribute is always resolved relative to ``src/``. -There is full flexibility in organizing the tests. -Multiple tests may be defined in a single file or they may be split in multiple files. -Similarly, several tests may share the same resources directory or they can simply have their own. - -Now it's time to run our first test: - -.. code:: bash - - ./bin/reframe -c tutorials/basics/hello/hello1.py -r - - -.. literalinclude:: listings/hello1.txt - :language: console - - -Perfect! We have verified that we have a functioning C compiler in our system. - -When ReFrame runs a test, it copies all its resources to a stage directory and performs all test-related operations (compilation, run, sanity checking etc.) from that directory. -On successful outcome of the test, the stage directory is removed by default, but interesting files are copied to an output directory for archiving and later inspection. -The prefixes of these directories are printed in the first section of the output. -Let's inspect what files ReFrame produced for this test: - -.. code-block:: console - - ls output/generic/default/builtin/HelloTest/ - -.. code-block:: none - - rfm_HelloTest_build.err rfm_HelloTest_build.sh rfm_HelloTest_job.out - rfm_HelloTest_build.out rfm_HelloTest_job.err rfm_HelloTest_job.sh - -ReFrame stores in the output directory of the test the build and run scripts it generated for building and running the code along with their standard output and error. -All these files are prefixed with ``rfm_``. - -ReFrame also generates a detailed JSON report for the whole regression testing session. -By default, this is stored inside the ``${HOME}/.reframe/reports`` directory and a new report file is generated every time ReFrame is run, but you can control this through the :option:`--report-file` command-line option. - -Here are the contents of the report file for our first ReFrame run: - - -.. code-block:: console - - cat ~/.reframe/reports/run-report.json - -.. literalinclude:: listings/run-report.json - - -More of "Hello, World!" ------------------------ - -We want to extend our test and run a C++ "Hello, World!" as well. -We could simply copy paste the ``hello1.py`` and change the source file extension to refer to the C++ source code. -But this duplication is something that we generally want to avoid. -ReFrame allows you to avoid this in several ways but the most compact is to define the new test as follows: - - -.. code-block:: console - - cat tutorials/basics/hello/hello2.py - - -.. literalinclude:: ../tutorials/basics/hello/hello2.py - :start-at: import reframe - - -This test extends the ``hello1.py`` test by defining the ``lang`` parameter with the :py:func:`~reframe.core.pipeline.RegressionMixin.parameter` built-in. -This parameter will cause as many instantiations as parameter values available, each one setting the :attr:`lang` attribute to one single value. -Hence, this example will create two test instances, one with ``lang='c'`` and another with ``lang='cpp'``. -The parameter is available as an attribute of the test instance and, in this example, we use it to set the extension of the source file. -However, at the class level, a test parameter holds all the possible values for itself, and this is only assigned a single value after the class is instantiated. -Therefore, the variable ``sourcepath``, which depends on this parameter, also needs to be set after the class instantiation. -The simplest way to do this would be to move the ``sourcepath`` assignment into the :func:`__init__` method as shown in the code snippet below, but this has some disadvantages when writing larger tests. - -.. code-block:: python - - def __init__(self): - self.sourcepath = f'hello.{self.lang}' - -For example, when writing a base class for a test with a large amount of code into the :func:`__init__` method, the derived class may want to do a partial override of the code in this function. -This would force us to understand the full implementation of the base class' :func:`__init__` despite that we may just be interested in overriding a small part of it. -Doable, but not ideal. -Instead, through pipeline hooks, ReFrame provides a mechanism to attach independent functions to execute at a given time before the data they set is required by the test. -This is exactly what we want to do here, and we know that the test sources are needed to compile the code. -Hence, we move the ``sourcepath`` assignment into a pre-compile hook. - -.. literalinclude:: ../tutorials/basics/hello/hello2.py - :start-at: @run_before('compile') - :end-at: self.sourcepath - -The use of hooks is covered in more detail later on, but for now, let's just think of them as a way to defer the execution of a function to a given stage of the test's pipeline. -By using hooks, any user could now derive from this class and attach other hooks (for example, adding some compiler flags) without having to worry about overriding the base method that sets the ``sourcepath`` variable. - -Let's run the test now: - - -.. code-block:: console - - ./bin/reframe -c tutorials/basics/hello/hello2.py -r - -.. literalinclude:: listings/hello2.txt - :language: console - -Oops! The C++ test has failed. -ReFrame complains that it does not know how to compile a C++ program. -Remember our discussion above that the default configuration of ReFrame defines a minimal programming environment named ``builtin`` which only knows of a ``cc`` compiler. -We will fix that in a moment, but before doing that it's worth looking into the failure information provided for the test. -For each failed test, ReFrame will print a short summary with information about the system partition and the programming environment that the test failed for, its job or process id (if any), the nodes it was running on, its stage directory, the phase that failed etc. - -When a test fails its stage directory is kept intact, so that users can inspect the failure and try to reproduce it manually. -In this case, the stage directory contains only the "Hello, World" source files, since ReFrame could not produce a build script for the C++ test, as it doesn't know to compile a C++ program for the moment. - -.. code-block:: console - - ls stage/generic/default/builtin/HelloMultiLangTest_cpp - - -.. code-block:: none - - hello.c hello.cpp - - -Let's go on and fix this failure by defining a new system and programming environments for the machine we are running on. -For this we need to create our own configuration file. - -.. code-block:: console - - vi tutorials/config/tresa.py - -Here is what we need to type: - -.. literalinclude:: ../tutorials/config/tresa.py - -We define a system named ``tresa`` that has one partition named ``default``. -This partition makes no use of any `workload manager `__, but instead launches any jobs locally as OS processes. -Two programming environments are relevant for that partition, namely ``gnu`` and ``clang``, which are defined in the section :data:`environments` of the configuration file. -The ``gnu`` programming environment provides GCC 12, whereas the ``clang`` one provides the Clang compiler from the system. -Notice, how you can define the actual commands for invoking the C, C++ and Fortran compilers in each programming environment. -As soon as a programming environment defines the different compilers, ReFrame will automatically pick the right compiler based on the source file extension. -In addition to C, C++ and Fortran programs, ReFrame will recognize the ``.cu`` extension as well and will try to invoke the ``nvcc`` compiler for CUDA programs. -Note also that we set the :attr:`~config.environments.target_systems` for each environment definition. -This restricts the definition of the environment being defined to the specified systems only. -ReFrame will always pick the definition that is a closest match for the current system. -Restricting the environment definitions is generally a good practice if you plan to define multiple systems in multiple configuration files, as ReFrame would otherwise complain that an environment is redefined. -On the other hand, if you want to provide generic definitions of environments that are valid for multiple systems, you may skip that. -This is what the builtin configuration of ReFrame does for its generic ``builtin`` environment. - -Finally, the new system that we defined may be identified by the hostname ``tresa`` (see the :attr:`~config.systems.hostnames` systems configuration parameter) and it will not use any environment modules system (see the :attr:`~config.systems.modules_system` configuration parameter). -The :attr:`~config.systems.hostnames` attribute will help ReFrame to automatically pick the right configuration when running on it. -Notice, how the ``generic`` system matches any hostname, so that it acts as a fallback system. - -.. note:: - - Multiple systems may defined in a configuration file, in which case they are tried in order and the first match is picked. - This means that the systems whose ``hostnames`` patterns are more generic, they should go to the end of the list. - -The :doc:`configure` page describes the configuration file in more detail and the :doc:`config_reference` provides a complete reference guide of all the configuration options of ReFrame. - -Let's now rerun our "Hello, World!" tests: - - -.. code-block:: console - - ./bin/reframe -C tutorials/config/tresa.py -c tutorials/basics/hello/hello2.py -r - - -.. literalinclude:: listings/hello2_tresa.txt - :language: console - -Notice how the same tests are now tried with both the ``gnu`` and ``clang`` programming environments, without having to touch them at all! -That's one of the powerful features of ReFrame and we shall see later on, how easily we can port our tests to an HPC cluster with minimal changes. -In order to instruct ReFrame to use our configuration file, we use the :option:`-C` command line option. -Since we don't want to type it throughout the tutorial, we could set the :envvar:`RFM_CONFIG_FILES` environment variable, which takes a colon-separated list of configuration files that ReFrame will load. -We will take advantage of multiple configuration files later in the tutorial. - -.. code-block:: console - - export RFM_CONFIG_FILES=$(pwd)/tutorials/config/tresa.py - - -.. tip:: - If our configuration file was named ``settings.py`` and we did not intend to use multiple configuration files in the same directory, we could also set the :envvar:`RFM_CONFIG_PATH` environment variable. - - -A Multithreaded "Hello, World!" -------------------------------- - -We extend our C++ "Hello, World!" example to print the greetings from multiple threads: - - -.. code-block:: console - - cat tutorials/basics/hellomp/src/hello_threads.cpp - - -.. literalinclude:: ../tutorials/basics/hellomp/src/hello_threads.cpp - :language: cpp - :start-at: #include - -This program takes as argument the number of threads it will create and it uses ``std::thread``, which is a C++11 addition, meaning that we will need to pass ``-std=c++11`` to our compilers. -Here is the corresponding ReFrame test, where the new concepts introduced are highlighted: - -.. code-block:: console - - cat tutorials/basics/hellomp/hellomp1.py - - -.. literalinclude:: ../tutorials/basics/hellomp/hellomp1.py - :start-at: import reframe - :emphasize-lines: 10-10, 13-18 - - -ReFrame delegates the compilation of a test to a :attr:`~reframe.core.pipeline.RegressionTest.build_system`, which is an abstraction of the steps needed to compile the test. -Build systems take also care of interactions with the programming environment if necessary. -Compilation flags are a property of the build system. -If not explicitly specified, ReFrame will try to pick the correct build system (e.g., CMake, Autotools etc.) by inspecting the test resources, but in cases as the one presented here where we need to set the compilation flags, we need to specify a build system explicitly. -In this example, we instruct ReFrame to compile a single source file using the ``-std=c++11 -pthread -Wall`` compilation flags. -However, the flag ``-pthread`` is only needed to compile applications using ``std::thread`` with the GCC and Clang compilers. -Hence, since this flag may not be valid for other compilers, we need to include it only in the tests that use either GCC or Clang. -Similarly to the ``lang`` parameter in the previous example, the information regarding which compiler is being used is only available after the class is instantiated (after completion of the ``setup`` pipeline stage), so we also defer the addition of this optional compiler flag with a pipeline hook. -In this case, we set the :func:`set_compile_flags` hook to run before the ReFrame pipeline stage ``compile``. - -.. note:: - - The pipeline hooks, as well as the regression test pipeline itself, are covered in more detail later on in the tutorial. - - -In this example, the generated executable takes a single argument which sets the number of threads to be used. -The options passed to the test's executable can be set through the :attr:`executable_opts ` variable, which in this case is set to ``'16'``. - -Let's run the test now: - -.. code-block:: console - - ./bin/reframe -c tutorials/basics/hellomp/hellomp1.py -r - - -.. literalinclude:: listings/hellomp1.txt - :language: console - - -Everything looks fine, but let's inspect the actual output of one of the tests: - - -.. code-block:: console - - cat output/catalina/default/clang/HelloThreadedTest/rfm_HelloThreadedTest_job.out - - -.. code-block:: none - - [[[[ 8] Hello, World! - 1] Hello, World! - 5[[0[ 7] Hello, World! - ] ] Hello, World! - [ Hello, World! - 6[] Hello, World! - 9] Hello, World! - 2 ] Hello, World! - 4] [[10 3] Hello, World! - ] Hello, World! - [Hello, World! - 11] Hello, World! - [12] Hello, World! - [13] Hello, World! - [14] Hello, World! - [15] Hello, World! - - -Not exactly what we were looking for! -In the following we write a more robust sanity check that can catch this havoc. - - ------------------------------ -More advanced sanity checking ------------------------------ - -So far, we have seen only a ``grep``-like search for a string in the test's :attr:`~reframe.core.pipeline.RegressionTest.stdout`, but ReFrame's :attr:`@sanity_function` are much more capable than this. -In fact, one could practically do almost any operation in the output and process it as you would like before assessing the test's sanity. -In the following, we extend the sanity checking of the above multithreaded "Hello, World!" to assert that all the threads produce a greetings line. -See the highlighted lines below in the modified version of the :attr:`@sanity_function`. - -.. code-block:: console - - cat tutorials/basics/hellomp/hellomp2.py - - -.. literalinclude:: ../tutorials/basics/hellomp/hellomp2.py - :start-at: import reframe - :emphasize-lines: 22-24 - -This new :attr:`@sanity_function` counts all the pattern matches in the tests's :attr:`~reframe.core.pipeline.RegressionTest.stdout` and checks that this count matches the expected value. -The execution of the function :func:`assert_num_messages` is deferred to the ``sanity`` stage in the test's pipeline, after the executable has run and the :attr:`~reframe.core.pipeline.RegressionTest.stdout` file has been populated. -In this example, we have used the :func:`~reframe.utility.sanity.findall` utility function from the :mod:`~reframe.utility.sanity` module to conveniently extract the pattern matches. -This module provides a broad range of utility functions that can be used to compose more complex sanity checks. -However, note that the utility functions in this module are lazily evaluated expressions or `deferred expressions` which must be evaluated either implicitly or explicitly (see :doc:`deferrable_functions_reference`). - -Let's run this version of the test now and see if it fails: - -.. code-block:: console - - ./bin/reframe -c tutorials/basics/hellomp/hellomp2.py -r - -.. literalinclude:: listings/hellomp2.txt - :language: console - -As expected, only some of lines are printed correctly which makes the test fail. -To fix this test, we need to compile with ``-DSYNC_MESSAGES``, which will synchronize the printing of messages. - -.. code-block:: console - - cat tutorials/basics/hellomp/hellomp3.py - - -.. literalinclude:: ../tutorials/basics/hellomp/hellomp3.py - :start-at: import reframe - :emphasize-lines: 15 - - -.. _perftest-basics: - -Writing A Performance Test --------------------------- - -An important aspect of regression testing is checking for performance regressions. -In this example, we write a test that downloads the `STREAM `__ benchmark, compiles it, runs it and records its performance. -In the test below, we highlight the lines that introduce new concepts. - -.. code-block:: console - - cat tutorials/basics/stream/stream1.py - - -.. literalinclude:: ../tutorials/basics/stream/stream1.py - :start-at: import reframe - :emphasize-lines: 9-11,14-17,28- - -First of all, notice that we restrict the programming environments to ``gnu`` only, since this test requires OpenMP, which our installation of Clang does not have. -The next thing to notice is the :attr:`~reframe.core.pipeline.RegressionTest.prebuild_cmds` attribute, which provides a list of commands to be executed before the build step. -These commands will be executed from the test's stage directory. -In this case, we just fetch the source code of the benchmark. -For running the benchmark, we need to set the OpenMP number of threads and pin them to the right CPUs through the ``OMP_NUM_THREADS`` and ``OMP_PLACES`` environment variables. -You can set environment variables in a ReFrame test through the :attr:`~reframe.core.pipeline.RegressionTest.env_vars` dictionary. - -What makes a ReFrame test a performance test is the definition of at least one :ref:`performance function`. -Similarly to a test's :func:`@sanity_function`, a performance function is a member function decorated with the :func:`@performance_function` decorator that merely extracts or computes a performance metric from the test's output and associates it with a unit. -By default, every performance function defined in the test is assigned to a *performance variable* with the function's name. -A performance variable is a named quantity representing a performance metric that ReFrame will report on, log and can also check against a reference value. -The performance variables of a test are stored in the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary. -The keys are the names of the metrics, whereas the values are :ref:`performance functions `. -The :func:`@performance_function` decorator apart from turning an ordinary method into a "performance function", it also creates an entry in the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary. -The optional ``perf_key`` argument can be used to assign a different name to the newly created performance variable. - -In this example, we extract four performance variables, namely the memory bandwidth values for each of the "Copy", "Scale", "Add" and "Triad" sub-benchmarks of STREAM, where each of the performance functions use the :func:`~reframe.utility.sanity.extractsingle` utility function. -For each of the sub-benchmarks we extract the "Best Rate MB/s" column of the output (see below) and we convert that to a float. - -.. code-block:: none - - Function Best Rate MB/s Avg time Min time Max time - Copy: 24939.4 0.021905 0.021527 0.022382 - Scale: 16956.3 0.031957 0.031662 0.032379 - Add: 18648.2 0.044277 0.043184 0.046349 - Triad: 19133.4 0.042935 0.042089 0.044283 - - -Let's run the test now: - - -.. code-block:: console - - ./bin/reframe -c tutorials/basics/stream/stream1.py -r --performance-report - -The :option:`--performance-report` will generate a short report at the end of the run for each performance test that has run. -Additionally, as soon as a performance test finishes, the obtained performance for each of the metrics is immediately reported. -This is especially useful if you run long suites of performance exploration tests and you do not want to wait until the end of the run to have an overview of the obtained performance. - - -.. literalinclude:: listings/stream1.txt - :language: console - - ---------------------------------------------------- -Setting explicitly the test's performance variables ---------------------------------------------------- - -Users are allowed to manipulate the test's :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary directly. -This is useful to avoid code repetition or in cases that relying on decorated methods to populate the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` is impractical, e.g., creating multiple performance variables in a loop. - -You might have noticed that in our STREAM example above, all four performance functions are almost identical except for a small part of the regex pattern. -In the following example, we define a single performance function, :func:`extract_bw`, that can extract any of the requested bandwidth metrics, and we populate the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` ourselves in a pre-performance hook: - -.. code-block:: console - - cat tutorials/basics/stream/stream2.py - -.. literalinclude:: ../tutorials/basics/stream/stream2.py - :start-at: import reframe - :emphasize-lines: 28- - -As mentioned in the previous section the :func:`@performance_function ` decorator performs two tasks: - -1. It converts a test method to *performance function*, i.e., a function that is suitable for extracting a performance metric. -2. It updates the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary with the newly created performance function. - -In this example, we are only interested in the first functionality and that's why we redefine completely the test's :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` using the :func:`extract_bw` performance function. -If you are inheriting from a base test and you don't want to override completely its performance variables, you could call instead :py:func:`update` on :attr:`~reframe.core.pipeline.RegressionTest.perf_variables`. - -Finally, you can convert any arbitrary function or :doc:`deferred expression ` into a performance function by calling the :func:`~reframe.utility.sanity.make_performance_function` utility as shown below: - -.. code-block:: python - - @run_before('performance') - def set_perf_vars(self): - self.perf_variables = { - 'Copy': sn.make_performance_function( - sn.extractsingle(r'Copy:\s+(\S+)\s+.*', - self.stdout, 1, float), - 'MB/s' - ) - } - -Note that in this case, the newly created performance function is not assigned to a test's performance variable and you will have to do this independently. - ------------------------ -Adding reference values ------------------------ - -On its current state, the above STREAM performance test will simply extract and report the performance variables regardless of the actual performance values. -However, in some situations, it might be useful to check that the extracted performance values are within an expected range, and report a failure whenever a test performs below expectations. -To this end, ReFrame tests include the :attr:`~reframe.core.pipeline.RegressionTest.reference` variable, which enables setting references for each of the performance variables defined in a test and also set different references for different systems. -In the following example, we set the reference values for all the STREAM sub-benchmarks for the system we are currently running on. - -.. note:: - - Optimizing STREAM benchmark performance is outside the scope of this tutorial. - - -.. code-block:: console - - cat tutorials/basics/stream/stream3.py - - -.. literalinclude:: ../tutorials/basics/stream/stream3.py - :start-at: import reframe - :emphasize-lines: 18-25 - - -The performance reference tuple consists of the reference value, the lower and upper thresholds expressed as fractional numbers relative to the reference value, and the unit of measurement. -If any of the thresholds is not relevant, :class:`None` may be used instead. -Also, the units in this :attr:`~reframe.core.pipeline.RegressionTest.reference` variable are entirely optional, since they were already provided through the :attr:`@performance_function` decorator. - -If any obtained performance value is beyond its respective thresholds, the test will fail with a summary as shown below: - -.. code-block:: console - - ./bin/reframe -c tutorials/basics/stream/stream3.py -r --performance-report - -.. literalinclude:: listings/stream3_failure_only.txt - :language: console - ------------------------------- -Examining the performance logs ------------------------------- - -ReFrame has a powerful mechanism for logging its activities as well as performance data. -It supports different types of log channels and it can send data simultaneously in any number of them. -For example, performance data might be logged in files and at the same time being sent to Syslog or to a centralized log management server. -By default (i.e., starting off from the builtin configuration file), ReFrame sends performance data to files per test under the ``perflogs/`` directory: - -.. code-block:: none - - perflogs - └── catalina - └── default - ├── StreamTest.log - └── StreamWithRefTest.log - -ReFrame creates a log file per test per system and per partition and appends to it every time the test is run on that system/partition combination. -Let's inspect the log file from our last test: - -.. code-block:: console - - tail perflogs/catalina/default/StreamWithRefTest.log - -.. literalinclude:: listings/perflogs.txt - :language: console - -The format of this file is controlled by :attr:`~config.logging.handlers_perflog` logging configuration parameter and, by default, contains several information about the test. -For each test, all of its performance variables are logged along with their unit, the obtained value, the reference and the lower and upper threshold. -The default format is in CSV, so that it can be easily post-processed. -For this reason, a header is also printed to help identify the different fields. - -Since version 4.0, ReFrame is very cautious when generating this file: if a change is detected in the information that is being logged, ReFrame will not append to the file, but it will instead create a new one, saving the old file using the ``.h`` suffix, where ``N`` is an integer that is increased every time a new file is being created due to such changes. -Examples of changes in the logged information are when the log record format changes or a new performance metric is added, deleted or has its name changed. -This behavior guarantees that each log file is consistent and it will not break existing parsers. - -For more information on configuring performance logging in ReFrame as well as logging in general, you may refer to the :ref:`logging-config-reference` reference. - - -Porting The Tests to an HPC cluster ------------------------------------ - -It's now time to port our tests to an HPC cluster. -Obviously, HPC clusters are much more complex than our laptop or PC. -Usually there are many more compilers, the user environment is handled in a different way, and the way to launch the tests varies significantly, since you have to go through a workload manager in order to access the actual compute nodes. -Besides that, there might be multiple types of compute nodes that we would like to run our tests on, but each type might be accessed in a different way. -It is already apparent that porting even an as simple as a "Hello, World" test to such a system is not that straightforward. -As we shall see in this section, ReFrame makes that pretty easy. - --------------------------- -Adapting the configuration --------------------------- - -Our target system is the `Piz Daint `__ supercomputer at CSCS, but you can adapt the process to your target HPC system. -In ReFrame, all the details of the various interactions of a test with the system environment are handled transparently and are set up in its configuration file. -Let's create a new configuration file for Piz Daint: - - -.. literalinclude:: ../tutorials/config/daint.py - :start-at: site_configuration - - -First of all, we need to define a new system and set the list of hostnames that will help ReFrame identify it. -We also set the :attr:`~config.systems.modules_system` configuration parameter to instruct ReFrame that this system makes use of the `environment modules `__ for managing the user environment. -Then we define the system partitions that we want to test. -In this case, we define three partitions: - -1. the login nodes, -2. the multicore partition (2x Broadwell CPUs per node) and -3. the hybrid partition (1x Haswell CPU + 1x Pascal GPU). - -.. |srun| replace:: :obj:`srun` -.. _srun: https://slurm.schedmd.com/srun.html - -The login nodes are pretty much similar to the ``tresa:default`` partition which corresponded to our laptop: tests will be launched and run locally. -The other two partitions are handled by `Slurm `__ and parallel jobs are launched using the |srun|_ command. -Additionally, in order to access the different types of nodes represented by those partitions, users have to specify either ``-C mc`` or ``-C gpu`` options along with their account. -This is what we do exactly with the :attr:`~config.systems.partitions.access` partition configuration option. - -.. note:: - - System partitions in ReFrame do not necessarily correspond to real job scheduler partitions. - -Piz Daint's programming environment offers four compilers: Cray, GNU, Intel and NVIDIA. -We want to test all of them, so we include them in the :attr:`~config.systems.partitions.environs` lists. -Notice that we do not include Clang in the list, since there is no such compiler on this particular system. -On the other hand, we include a different version of the ``builtin`` environment, which corresponds to the default login environment without loading any modules. -It is generally useful to define such an environment so as to use it for tests that are running simple utilities and don't need to compile anything. - -Before looking into the definition of the new environments for the four compilers, it is worth mentioning the :attr:`~config.systems.partitions.max_jobs` parameter. -This parameter specifies the maximum number of ReFrame test jobs that can be simultaneously in flight. -ReFrame will try to keep concurrency close to this limit (but not exceeding it). -By default, this is set to ``8``, so you are advised to set it to a higher number if you want to increase the throughput of completed tests. - -The new environments are defined similarly to the ones we had for our local system, except that now we add also the :attr:`~config.environments.modules` parameter. -The :attr:`~config.environments.modules` parameter is a list of environment modules that needs to be loaded, in order to make available this compiler. - ------------------ -Running the tests ------------------ - -We are now ready to run our tests on Piz Daint. -We will only do so with the final versions of the tests from the previous section, which we will select using :option:`-n` option. - -.. code-block:: console - - export RFM_CONFIG_FILES=$(pwd)/tutorials/config/daint.py - ./bin/reframe -c tutorials/basics/ -R -n 'HelloMultiLangTest|HelloThreadedExtended2Test|StreamWithRefTest' --performance-report -r - -.. literalinclude:: listings/alltests_daint.txt - :language: console - -There it is! -Without any change in our tests, we could simply run them in a HPC cluster with all of its intricacies. -Notice how our original four tests expanded to more than 40 test cases on that particular HPC cluster! -One reason we could run immediately our tests on a new system was that we have not been restricting neither the valid system they can run nor the valid programming environments they can run with (except for the STREAM test). -Otherwise we would have to add ``daint`` and its corresponding programming environments in :attr:`valid_systems` and :attr:`valid_prog_environs` lists respectively. - -.. tip:: - - A quick way to try a test on a new system, if it's not generic, is to pass the :option:`--skip-system-check` and the :option:`--skip-prgenv-check` command line options which will cause ReFrame to skip any test validity checks for systems or programming environments. - -Although the tests remain the same, ReFrame has generated completely different job scripts for each test depending on where it was going to run. -Let's check the job script generated for the ``StreamWithRefTest``: - -.. code-block:: console - - cat output/daint/gpu/gnu/StreamWithRefTest/rfm_StreamWithRefTest_job.sh - -.. code-block:: bash - - #!/bin/bash - #SBATCH --job-name="rfm_StreamWithRefTest_job" - #SBATCH --ntasks=1 - #SBATCH --output=rfm_StreamWithRefTest_job.out - #SBATCH --error=rfm_StreamWithRefTest_job.err - #SBATCH --time=0:10:0 - #SBATCH -A csstaff - #SBATCH --constraint=gpu - module unload PrgEnv-cray - module load PrgEnv-gnu - export OMP_NUM_THREADS=4 - export OMP_PLACES=cores - srun ./StreamWithRefTest - -Whereas the exact same test running on our laptop was as simple as the following: - - -.. code-block:: bash - - #!/bin/bash - export OMP_NUM_THREADS=4 - export OMP_PLACES=cores - ./StreamWithRefTest - -In ReFrame, you don't have to care about all the system interaction details, but rather about the logic of your tests as we shall see in the next section. - - ------------------------------------------------------------ -Adapting a test to new systems and programming environments ------------------------------------------------------------ - -Unless a test is rather generic, you will need to make some adaptations for the system that you port it to. -In this case, we will adapt the STREAM benchmark so as to run it with multiple compiler and adjust its execution based on the target architecture of each partition. -Let's see and comment the changes: - -.. code-block:: console - - cat tutorials/basics/stream/stream4.py - -.. literalinclude:: ../tutorials/basics/stream/stream4.py - :start-at: import reframe - :emphasize-lines: 8, 27-41, 43-56 - -First of all, we need to add the new programming environments in the list of the supported ones. -Now there is the problem that each compiler has its own flags for enabling OpenMP, so we need to differentiate the behavior of the test based on the programming environment. -For this reason, we define the flags for each compiler in a separate dictionary (``flags`` variable) and we set them in the :func:`set_compiler_flags` pipeline hook. -We have first seen the pipeline hooks in the multithreaded "Hello, World!" example and now we explain them in more detail. -When ReFrame loads a test file, it instantiates all the tests it finds in it. -Based on the system ReFrame runs on and the supported environments of the tests, it will generate different test cases for each system partition and environment combination and it will finally send the test cases for execution. -During its execution, a test case goes through the *regression test pipeline*, which is a series of well defined phases. -Users can attach arbitrary functions to run before or after any pipeline stage and this is exactly what the :func:`set_compiler_flags` function is. -We instruct ReFrame to run this function before the test enters the ``compile`` stage and set accordingly the compilation flags. -The system partition and the programming environment of the currently running test case are available to a ReFrame test through the :attr:`~reframe.core.pipeline.RegressionTest.current_partition` and :attr:`~reframe.core.pipeline.RegressionTest.current_environ` attributes respectively. -These attributes, however, are only set after the first stage (``setup``) of the pipeline is executed, so we can't use them inside the test's constructor. - -We do exactly the same for setting the ``OMP_NUM_THREADS`` environment variables depending on the system partition we are running on, by attaching the :func:`set_num_threads` pipeline hook to the ``run`` phase of the test. -In that same hook we also set the :attr:`~reframe.core.pipeline.RegressionTest.num_cpus_per_task` attribute of the test, so as to instruct the backend job scheduler to properly assign CPU cores to the test. -In ReFrame tests you can set a series of task allocation attributes that will be used by the backend schedulers to emit the right job submission script. -The section :ref:`scheduler_options` of the :doc:`regression_test_api` summarizes these attributes and the actual backend scheduler options that they correspond to. - -For more information about the regression test pipeline and how ReFrame executes the tests in general, have a look at :doc:`pipeline`. - -.. note:: - - ReFrame tests are ordinary Python classes so you can define your own attributes as we do with :attr:`flags` and :attr:`cores` in this example. - -Let's run our adapted test now: - -.. code-block:: console - - ./bin/reframe -c tutorials/basics/stream/stream4.py -r --performance-report - - -.. literalinclude:: listings/stream4_daint.txt - :language: console - -Notice the improved performance of the benchmark in all partitions and the differences in performance between the different compilers. - -This concludes our introductory tutorial to ReFrame! diff --git a/docs/tutorial_build_automation.rst b/docs/tutorial_build_automation.rst deleted file mode 100644 index a2d9db5ca9..0000000000 --- a/docs/tutorial_build_automation.rst +++ /dev/null @@ -1,164 +0,0 @@ -========================================================== -Tutorial 5: Using Build Automation Tools As a Build System -========================================================== - -In this tutorial we will present how to use `Easybuild `__ and `Spack `__ as a build system for a ReFrame test. -The example uses the configuration file presented in :doc:`tutorial_basics`, which you can find in ``tutorials/config/settings.py``. -We also assume that the reader is already familiar with the concepts presented in the basic tutorial and has a working knowledge of EasyBuild and Spack. - - -Using EasyBuild to Build the Test Code --------------------------------------- - -.. versionadded:: 3.5.0 - - -Let's consider a simple ReFrame test that installs ``bzip2-1.0.6`` given the easyconfig `bzip2-1.0.6.eb `__ and checks that the installed version is correct. -The following code block shows the check, highlighting the lines specific to this tutorial: - -.. literalinclude:: ../tutorials/build_systems/easybuild/eb_test.py - :start-at: import reframe - :emphasize-lines: 12,14-17,19-21 - - -The test looks pretty standard except for the highlighted blocks. -Let's have a look first to the block in the :class:`BZip2Check` class. - -The first thing is to specify that the EasyBuild build system will be used. -This is done by setting :attr:`~reframe.core.pipeline.RegressionTest.build_system` to ``'EasyBuild'``. -Then, the software to be installed is passed as a list to :attr:`~reframe.core.buildsystems.EasyBuild.easyconfigs`. -Here only one easyconfig is given, but more than one can be passed. -Finally, through :attr:`~reframe.core.buildsystems.EasyBuild.options`, command line options can be passed to the ``eb`` executable. -In this test we pass ``-f`` to make sure that ``bzip2`` will be built even if the module already exists externally. - -For this test, ReFrame generates the following command to build and install the easyconfig: - -.. code-block:: console - - export EASYBUILD_BUILDPATH={stagedir}/easybuild/build - export EASYBUILD_INSTALLPATH={stagedir}/easybuild - export EASYBUILD_PREFIX={stagedir}/easybuild - export EASYBUILD_SOURCEPATH={stagedir}/easybuild - eb bzip2-1.0.6.eb -f - -ReFrame will keep all the files generated by EasyBuild (sources, temporary files, installed software and the corresponding modules) under the test's stage directory. -For this reason it sets the relevant EasyBuild environment variables. - -.. tip:: - - Users may set the EasyBuild prefix to a different location by setting the :attr:`~reframe.core.buildsystems.EasyBuild.prefix` attribute of the build system. - This allows you to have the built software installed upon successful completion of the build phase, but if the test fails in a later stage (sanity, performance), the installed software will not be cleaned up automatically. - -.. note:: - - ReFrame assumes that the ``eb`` executable is available on the system where the compilation is run (typically the local host where ReFrame is executed). - - -Now that we know everything related to building and installing the code, we can move to the part dealing with running it. -To run the code, the generated modules need to be loaded in order to make the software available. -The modules can be accessed through :attr:`~reframe.core.buildsystems.Easybuild.generated_modules`, however, they are available only after EasyBuild completes the installation. -This means that :attr:`~reframe.core.pipeline.RegressionTest.modules` can be set only after the build phase finishes. -For that, we can set :attr:`~reframe.core.pipeline.RegressionTest.modules` in a class method wrapped by the :py:func:`~reframe.core.RegressionTest.run_before` built-in, specifying the ``run`` phase. -This test will then run the following commands: - -.. code-block:: console - - module load bzip/1.0.6 - bzip2 --help - - --------------------------- -Packaging the installation --------------------------- - -The EasyBuild build system offers a way of packaging the installation via EasyBuild's packaging support. -To use this feature, `the FPM package manager `__ must be available. -By setting the dictionary :attr:`~reframe.core.buildsystems.Easybuild.package_opts` in the test, ReFrame will pass ``--package-{key}={val}`` to the EasyBuild invocation. -For instance, the following can be set to package the installations as an rpm file: - -.. code-block:: python - - self.keep_files = ['easybuild/packages'] - self.build_system.package_opts = { - 'type': 'rpm', - } - -The packages are generated by EasyBuild in the stage directory. -To retain them after the test succeeds, :attr:`~reframe.core.pipeline.RegressionTest.keep_files` needs to be set. - - - -Using Spack to Build the Test Code ----------------------------------- - - -.. versionadded:: 3.6.1 - - -This example is the equivalent to the previous one, except that it uses Spack to build ``bzip2``. -Here is the test's code: - -.. literalinclude:: ../tutorials/build_systems/spack/spack_test.py - :start-at: import reframe - - -When :attr:`~reframe.core.pipeline.RegressionTest.build_system` is set to ``'Spack'``, ReFrame will leverage Spack environments in order to build the test code. -By default, ReFrame will create a new Spack environment in the test's stage directory and add the requested :attr:`~reframe.core.buildsystems.Spack.specs` to it. - -.. note:: - Optional spec attributes, such as ``target`` and ``os``, should be specified in :attr:`~reframe.core.buildsystems.Spack.specs` and not as install options in :attr:`~reframe.core.buildsystems.Spack.install_opts`. - -You can set Spack configuration options for the new environment with the :attr:`~reframe.core.buildsystems.Spack.config_opts` attribute. These options take precedence over Spack's ``spack.yaml`` defaults. - -Users may also specify an existing Spack environment by setting the :attr:`~reframe.core.buildsystems.Spack.environment` attribute. -In this case, ReFrame treats the environment as a *test resource* so it expects to find it under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. - -As with every other test, ReFrame will copy the test's resources to its stage directory before building it. -ReFrame will then activate the generated environment (either the one provided by the user or the one generated by ReFrame), add the given specs using the ``spack add`` command and, finally, install the packages in the environment. -Here is what ReFrame generates as a build script for this example: - -.. code:: bash - - spack env create -d rfm_spack_env - spack -e rfm_spack_env config add "config:install_tree:root:opt/spack" - spack -e rfm_spack_env add bzip2@1.0.6 - spack -e rfm_spack_env install - -As you might have noticed ReFrame expects that Spack is already installed on the system. -The packages specified in the environment and the tests will be installed in the test's stage directory, where the environment is copied before building. -Here is the stage directory structure: - -.. code:: console - - stage/generic/default/builtin/BZip2SpackCheck/ - ├── rfm_spack_env - │   ├── spack - │   │   └── opt - │   │      └── spack - │   │      ├── bin - │   │      └── darwin-catalina-skylake - │   ├── spack.lock - │   └── spack.yaml - ├── rfm_BZip2SpackCheck_build.err - ├── rfm_BZip2SpackCheck_build.out - ├── rfm_BZip2SpackCheck_build.sh - ├── rfm_BZip2SpackCheck_job.err - ├── rfm_BZip2SpackCheck_job.out - └── rfm_BZip2SpackCheck_job.sh - - -Finally, here is the generated run script that ReFrame uses to run the test, once its build has succeeded: - -.. code-block:: bash - - #!/bin/bash - spack env create -d rfm_spack_env - eval `spack -e rfm_spack_env load --sh bzip2@1.0.6` - bzip2 --help - -From this point on, sanity and performance checking are exactly identical to any other ReFrame test. - -.. tip:: - - While developing a test using Spack or EasyBuild as a build system, it can be useful to run ReFrame with the :option:`--keep-stage-files` and :option:`--dont-restage` options to prevent ReFrame from removing the test's stage directory upon successful completion of the test. - For this particular type of test, these options will avoid having to rebuild the required package dependencies every time the test is retried. diff --git a/docs/tutorial_deps.rst b/docs/tutorial_deps.rst deleted file mode 100644 index 5f82acc3ef..0000000000 --- a/docs/tutorial_deps.rst +++ /dev/null @@ -1,209 +0,0 @@ -=============================================== -Tutorial 3: Using Dependencies in ReFrame Tests -=============================================== - -.. versionadded:: 2.21 - - -A ReFrame test may define dependencies to other tests. -An example scenario is to test different runtime configurations of a benchmark that you need to compile, or run a scaling analysis of a code. -In such cases, you don't want to download and rebuild your test for each runtime configuration. -You could have a test where only the sources are fetched, and which all build tests would depend on. -And, similarly, all the runtime tests would depend on their corresponding build test. -This is the approach we take with the following example, that fetches, builds and runs several `OSU benchmarks `__. -We first create a basic run-only test, that fetches the benchmarks: - -.. code-block:: console - - cat tutorials/deps/osu_benchmarks.py - - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSUDownloadTest - -This test doesn't need any specific programming environment, so we simply pick the ``builtin`` environment in the ``login`` partition. -The build tests would then copy the benchmark code and build it for the different programming environments: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSUBuildTest - -The only new thing that comes in with the :class:`OSUBuildTest` test is the following: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSUBuildTest.inject_dependencies - -Here we tell ReFrame that this test depends on a test named :class:`OSUDownloadTest`. -This test may or may not be defined in the same test file; all ReFrame needs is the test name. -The :func:`depends_on() ` function will create dependencies between the individual test cases of the :class:`OSUBuildTest` and the :class:`OSUDownloadTest`, such that all the test cases of :class:`OSUBuildTest` will depend on the outcome of the :class:`OSUDownloadTest`. -This behaviour can be changed, but it is covered in detail in :doc:`dependencies`. -You can create arbitrary test dependency graphs, but they need to be acyclic. -If ReFrame detects cyclic dependencies, it will refuse to execute the set of tests and will issue an error pointing out the cycle. - -A ReFrame test with dependencies will execute, i.e., enter its "setup" stage, only after *all* of its dependencies have succeeded. -If any of its dependencies fails, the current test will be marked as failure as well. - -The next step for the :class:`OSUBuildTest` is to set its :attr:`sourcesdir` to point to the source code that was fetched by the :class:`OSUDownloadTest`. -This is achieved with the following specially decorated function: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSUBuildTest.set_sourcedir - -The :func:`@require_deps ` decorator binds each argument of the decorated function to the corresponding target dependency. -In order for the binding to work correctly the function arguments must be named after the target dependencies. -Referring to a dependency only by the test's name is not enough, since a test might be associated with multiple programming environments. -For this reason, each dependency argument is actually bound to a function that accepts as argument the name of the target partition and target programming environment. -If no arguments are passed, the current programming environment is implied, such that ``OSUDownloadTest()`` is equivalent to ``OSUDownloadTest(self.current_environ.name, self.current_partition.name)``. -In this case, since both the partition and environment of the target dependency do not match those of the current test, we need to specify both. - -This call returns the actual test case of the dependency that has been executed. -This allows you to access any attribute from the target test, as we do in this example by accessing the target test's stage directory, which we use to construct the sourcesdir of the test. - -For the next test we need to use the OSU benchmark binaries that we just built, so as to run the MPI ping-pong benchmark. -Here is the relevant part: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :start-after: # rfmdocstart: osupingpong - :end-before: # rfmdocend: osupingpong - -First, since we will have multiple similar benchmarks, we move all the common functionality to the :class:`OSUBenchmarkTestBase` base class. -Again nothing new here; we are going to use two nodes for the benchmark and we set :attr:`sourcesdir ` to ``None``, since none of the benchmark tests will use any additional resources. -As done previously, we define the dependencies with the following: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSUBenchmarkTestBase.set_dependencies - -Here we tell ReFrame that this test depends on a test named :class:`OSUBuildTest` "by environment." -This means that the test cases of this test will only depend on the test cases of the :class:`OSUBuildTest` that use the same environment; -partitions may be different. - -The next step for the :class:`OSULatencyTest` is to set its executable to point to the binary produced by the :class:`OSUBuildTest`. -This is achieved with the following specially decorated function: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSULatencyTest.set_executable - -This concludes the presentation of the :class:`OSULatencyTest` test. The :class:`OSUBandwidthTest` is completely analogous. - -The :class:`OSUAllreduceTest` shown below is similar to the other two, except that it is parameterized. -It is essentially a scalability test that is running the ``osu_allreduce`` executable created by the :class:`OSUBuildTest` for 2, 4, 8 and 16 nodes. - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - :pyobject: OSUAllreduceTest - -The full set of OSU example tests is shown below: - -.. literalinclude:: ../tutorials/deps/osu_benchmarks.py - -Notice that the order in which dependencies are defined in a test file is irrelevant. -In this case, we define :class:`OSUBuildTest` at the end. -ReFrame will make sure to properly sort the tests and execute them. - -Here is the output when running the OSU tests with the asynchronous execution policy: - -.. code-block:: console - - ./bin/reframe -c tutorials/deps/osu_benchmarks.py -r - -.. literalinclude:: listings/osu_bench_deps.txt - :language: console - -Before starting running the tests, ReFrame topologically sorts them based on their dependencies and schedules them for running using the selected execution policy. -With the serial execution policy, ReFrame simply executes the tests to completion as they "arrive," since the tests are already topologically sorted. -In the asynchronous execution policy, tests are spawned and not waited for. -If a test's dependencies have not yet completed, it will not start its execution immediately. - -ReFrame's runtime takes care of properly cleaning up the resources of the tests respecting dependencies. -Normally when an individual test finishes successfully, its stage directory is cleaned up. -However, if other tests are depending on this one, this would be catastrophic, since most probably the dependent tests would need the outcome of this test. -ReFrame fixes that by not cleaning up the stage directory of a test until all its dependent tests have finished successfully. - -When selecting tests using the test filtering options, such as the :option:`-t`, :option:`-n` etc., ReFrame will automatically select any dependencies of these tests as well. -For example, if we select only the :class:`OSULatencyTest` for running, ReFrame will also select the :class:`OSUBuildTest` and the :class:`OSUDownloadTest`: - -.. code-block:: console - - ./bin/reframe -c tutorials/deps/osu_benchmarks.py -n OSULatencyTest -l - -.. literalinclude:: listings/osu_latency_list.txt - :language: console - -Finally, when ReFrame cannot resolve a dependency of a test, it will issue a warning and skip completely all the test cases that recursively depend on this one. -In the following example, we restrict the run of the :class:`OSULatencyTest` to the ``daint:gpu`` partition. -This is problematic, since its dependencies cannot run on this partition and, particularly, the :class:`OSUDownloadTest`. -As a result, its immediate dependency :class:`OSUBuildTest` will be skipped, which will eventually cause all combinations of the :class:`OSULatencyTest` to be skipped. - -.. code-block:: console - - ./bin/reframe -c tutorials/deps/osu_benchmarks.py --system=daint:gpu -n OSULatencyTest -l - -.. literalinclude:: listings/osu_latency_unresolved_deps.txt - :language: console - - -Listing Dependencies --------------------- - -As shown in the listing of :class:`OSULatencyTest` before, the full dependency chain of the test is listed along with the test. -Each target dependency is printed in a new line prefixed by the ``^`` character and indented proportionally to its level. -If a target dependency appears in multiple paths, it will only be listed once. - -The default test listing will list the dependencies at the test level or the *conceptual* dependencies. -ReFrame generates multiple test cases from each test depending on the target system configuration. -We have seen in the :doc:`tutorial_basics` already how the STREAM benchmark generated many more test cases when it was run in a HPC system with multiple partitions and programming environments. -These are the *actual* depedencies and form the actual test case graph that will be executed by the runtime. -The mapping of a test to its concrete test cases that will be executed on a system is called *test concretization*. -You can view the exact concretization of the selected tests with ``--list=concretized`` or simply ``-lC``. -Here is how the OSU benchmarks of this tutorial are concretized on the system ``daint``: - - -.. code-block:: console - - ./bin/reframe -c tutorials/deps/osu_benchmarks.py -lC - -.. literalinclude:: listings/osu_bench_list_concretized.txt - :language: console - -Notice how the various test cases of the run benchmarks depend on the corresponding test cases of the build tests. - -The concretization of test cases changes if a specifc partition or programming environment is passed from the command line or, of course, if the test is run on a different system. -If we scope our programming environments to ``gnu`` and ``builtin`` only, ReFrame will generate 8 test cases only instead of 22: - -.. note:: - - If we do not select the ``builtin`` environment, we will end up with a dangling dependency as in the example above and ReFrame will skip all the dependent test cases. - - -.. code-block:: console - - ./bin/reframe -c tutorials/deps/osu_benchmarks.py -n OSULatencyTest -L -p builtin -p gnu - -.. literalinclude:: listings/osu_bench_list_concretized_gnu.txt - :language: console - - -To gain a deeper understanding on how test dependencies work in Reframe, please refer to :doc:`dependencies`. - - -.. _param_deps: - -Depending on Parameterized Tests --------------------------------- - -As shown earlier in this section, tests define their dependencies by referencing the target tests by their unique name. -This is straightforward when referring to regular tests, where their name matches the class name, but it becomes cumbersome trying to refer to a parameterized tests, since no safe assumption should be made as of the variant number of the test or how the parameters are encoded in the name. -In order to safely and reliably refer to a parameterized test, you should use the :func:`~reframe.core.pipeline.RegressionMixin.get_variant_nums` and :func:`~reframe.core.pipeline.RegressionMixin.variant_name` class methods as shown in the following example: - -.. literalinclude:: ../tutorials/deps/parameterized.py - :emphasize-lines: 37- - -In this example, :class:`TestB` depends only on selected variants of :class:`TestA`. -The :func:`get_variant_nums` method accepts a set of key-value pairs representing the target test parameters and selector functions and returns the list of the variant numbers that correspond to these variants. -Using the :func:`variant_name` subsequently, we can get the actual name of the variant. - - -.. code-block:: console - - ./bin/reframe -c tutorials/deps/parameterized.py -l - -.. literalinclude:: listings/param_deps_list.txt - :language: console diff --git a/docs/tutorial_fixtures.rst b/docs/tutorial_fixtures.rst deleted file mode 100644 index 517dc21908..0000000000 --- a/docs/tutorial_fixtures.rst +++ /dev/null @@ -1,155 +0,0 @@ -Tutorial 4: Using Test Fixtures -=============================== - -.. versionadded:: 3.9.0 - -A fixture in ReFrame is a test that manages a resource of another test. -Fixtures can be chained to create essentially a graph of dependencies. -Similarly to test dependencies, the test that uses the fixture will not execute until its fixture has executed. -In this tutorial, we will rewrite the OSU benchmarks example presented in :doc:`tutorial_deps` using fixtures. -We will cover only the basic concepts of fixtures that will allow you to start using them in your tests. -For the full documentation of the test fixtures, you should refer to the :doc:`regression_test_api` documentation. - -The full example of the OSU benchmarks using test fixtures is shown below with the relevant parts highlighted: - -.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py - :start-at: import reframe - -Let's start from the leaf tests, i.e. the tests that execute the benchmarks (:class:`osu_latency_test`, :class:`osu_bandwidth_test` and :class:`osu_allreduce_test`). -As in the dependencies example, all these tests derive from the :class:`OSUBenchmarkTestBase`, where we define a fixture that will take care of generating the binaries of the tests: - -.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py - :start-at: osu_binaries = fixture - :end-at: osu_binaries = fixture - -A test defines a fixture using the :func:`~reframe.core.pipeline.RegressionMixin.fixture` builtin and assigns it a name by assigning the return value of the builtin to a test variable, here ``osu_binaries``. -This name will be used later to access the resource managed by the fixture. - -As stated previously, a fixture is another full-fledged ReFrame test, here the :class:`build_osu_benchmarks` which will take care of building the OSU benchmarks. -Each fixture is associated with a scope. -This practically indicates at which level a fixture is shared with other tests. -There are four fixture scopes, which are listed below in decreasing order of generality: - -- ``session``: A fixture with this scope will be executed once per ReFrame run session and will be shared across the whole run. -- ``partition``: A fixture with this scope will be executed once per partition and will be shared across all tests that run in that partition. -- ``environment``: A fixture with this scope will be executed once per partition and environment combination and will be shared across all tests that run with this partition and environment combination. -- ``test``: A fixture with this scope is private to the test and will be executed for each test case. - -In this example, we need to build once the OSU benchmarks for each partition and environment combination, so we use the ``environment`` scope. - -Accessing the fixture is very straightforward. -The fixture's result is accessible after the *setup* pipeline stage through the corresponding variable in the test that is defining it. -Since a fixture is a standard ReFrame test, you can access any information of the test. -The individual benchmarks do exactly that: - -.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py - :pyobject: osu_latency_test.prepare_run - :emphasize-lines: 4-5 - -Here we construct the final executable path by accessing the standard :attr:`~reframe.core.pipeline.RegressionTest.stagedir` attribute of the test as well as the custom-defined :attr:`build_prefix` variable of the :class:`build_osu_benchmarks` fixture. - -Let's inspect now the :class:`build_osu_benchmarks` fixture: - -.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py - :pyobject: build_osu_benchmarks - :emphasize-lines: 6,11,14 - -It is obvious that it is a normal ReFrame test except that it does not need to be decorated with the :func:`@simple_test ` decorator. -This means that the test will only be executed if it is a fixture of another test. -If it was decorated, it would be executed both as a standalone test and as a fixture of another test. -Another detail is that this test does not define the :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs` variables. -Fixtures inherit those variables from the test that owns them depending on the scope. - -Similarly to :class:`OSUBenchmarkTestBase`, this test uses a fixture that fetches the OSU benchmarks sources. -We could fetch the OSU benchmarks in this test, but we choose to separate the two primarily for demonstration purposes, but it would also make sense in cases that the data fetch is too slow. - -The ``osu_benchmarks`` fixture is defined at session scope, since we only need to download the benchmarks once for the whole session: - -.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py - :start-at: osu_benchmarks = fixture - :end-at: osu_benchmarks = fixture - -The rest of the test is very straightforward. - -Let's inspect the last fixture, the :class:`fetch_osu_benchmarks`: - -.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py - :pyobject: fetch_osu_benchmarks - :emphasize-lines: 8 - -There is nothing special to this test -- it is just an ordinary test -- except that we force it to execute locally by setting its :attr:`~reframe.core.pipeline.RegressionTest.local` variable. -The reason for that is that a fixture at session scope can execute with any partition/environment combination, so ReFrame could have to spawn a job in case it has chosen a remote partition to launch this fixture on. -For this reason, we simply force it to execute locally regardless of the chosen partition. - -It is now time to run the new tests, but let us first list them: - -.. code-block:: bash - - reframe -c tutorials/fixtures/osu_benchmarks.py -l - -.. literalinclude:: listings/osu_bench_fixtures_list.txt - :language: console - - -Notice how the :class:`build_osu_benchmarks` fixture is populated three times, once for each partition and environment combination, and the :class:`fetch_osu_benchmarks` is generated only once. -The following figure shows visually the conceptual dependencies of the :class:`osu_bandwidth_test`. - -.. figure:: _static/img/fixtures-conceptual-deps.svg - :align: center - - :sub:`Expanded fixtures and dependencies for the OSU benchmarks example.` - -A *scope* part is added to the base name of the fixture, which in this figure is indicated with red color. - -Under the hood, fixtures use the test dependency mechanism which is described in :doc:`dependencies`. -The dependencies listed by default and shown in the previous figure are conceptual. -Depending on the available partitions and environments, tests and fixtures can be concretized differently. -Fixtures in particular are also more flexible in the way they can be concretized depending on their scope. -The following listing and figure show the concretization of the :class:`osu_bandwidth_test`: - -.. code-block:: bash - - reframe -c tutorials/fixtures/osu_benchmarks.py -n osu_bandwidth_test -lC - -.. literalinclude:: listings/osu_bandwidth_concretized_daint.txt - :language: console - - -.. figure:: _static/img/fixtures-actual-deps.svg - :align: center - - :sub:`The actual dependencies for the OSU benchmarks example using fixtures.` - -The first thing to notice here is how the individual test cases of :class:`osu_bandwidth_test` depend only the specific fixtures for their scope: -when :class:`osu_bandwidth_test` runs on the ``daint:gpu`` partition using the ``gnu`` compiler it will only depend on the :class:`build_osu_benchmarks~daint:gpu+gnu` fixture. -The second thing to notice is where the :class:`fetch_osu_benchmarks~daint` fixture will run. -Since this is a *session* fixture, ReFrame has arbitrarily chosen to run it on ``daint:gpu`` using the ``gnu`` environment. -A session fixture can run on any combination of valid partitions and environments. -The following listing and figure show how the test dependency DAG is concretized when we scope the valid programming environments from the command line using ``-p nvidia``. - -.. code-block:: bash - - reframe -c tutorials/fixtures/osu_benchmarks.py -n osu_bandwidth_test -lC -p nvidia - -.. literalinclude:: listings/osu_bandwidth_concretized_daint_nvidia.txt - :language: console - -.. figure:: _static/img/fixtures-actual-deps-scoped.svg - :align: center - - :sub:`The dependency graph concretized for the 'nvidia' environment only.` - - -Notice how the :class:`fetch_osu_benchmarks~daint` fixture is selected to run in the only valid partition/environment combination. -This is an important difference compared to the same example written using raw dependencies in :doc:`dependencies`, in which case in order not to have unresolved dependencies, we would need to specify the valid programming environment of the test that fetches the sources. -Fixtures do not need that, since you can impose less strict constraints by setting their scope accordingly. - -Finally, let's run all the benchmarks at once: - -.. literalinclude:: listings/osu_bench_fixtures_run.txt - :language: console - -.. tip:: - A reasonable question is how to choose between fixtures and dependencies? - - The rule of thumb is use fixtures if your test needs to use any resource of the target test and use dependencies if you simply want to impose an order of execution for your tests. diff --git a/docs/tutorial_flux.rst b/docs/tutorial_flux.rst deleted file mode 100644 index b1bf2af21e..0000000000 --- a/docs/tutorial_flux.rst +++ /dev/null @@ -1,108 +0,0 @@ -======================================== -Tutorial 7: The Flux Framework Scheduler -======================================== - -This is a tutorial that will show how to use refame with `Flux -Framework `__. First, build the -container here from the root of reframe. - -.. code:: bash - - $ docker build -f tutorials/flux/Dockerfile -t flux-reframe . - -Then shell inside, optionally binding the present working directory if -you want to develop. - -.. code:: bash - - $ docker run -it -v $PWD:/code flux-reframe - $ docker run -it flux-reframe - -Note that if you build the local repository, you’ll need to bootstrap -and install again, as we have over-written the bin! - -.. code:: bash - - ./bootstrap.sh - -And then reframe will again be in the local ``bin`` directory: - -.. code:: bash - - # which reframe - /code/bin/reframe - -Then we can run ReFrame with the custom config `config.py `__ -for flux. - -.. code:: bash - - # What tests are under tutorials/flux? - $ cd tutorials/flux - $ reframe -c . -C settings.py -l - -.. code:: console - - [ReFrame Setup] - version: 4.0.0-dev.1 - command: '/code/bin/reframe -c tutorials/flux -C tutorials/flux/settings.py -l' - launched by: root@b1f6650222bc - working directory: '/code' - settings file: 'tutorials/flux/settings.py' - check search path: '/code/tutorials/flux' - stage directory: '/code/stage' - output directory: '/code/output' - - [List of matched checks] - - EchoRandTest /66b93401 - Found 1 check(s) - - Log file(s) saved in '/tmp/rfm-ilqg7fqg.log' - -This also works - -.. code:: bash - - $ reframe -c tutorials/flux -C tutorials/flux/settings.py -l - -And then to run tests, just replace ``-l`` (for list) with ``-r`` or -``--run`` (for run): - -.. code:: bash - - $ reframe -c tutorials/flux -C tutorials/flux/settings.py --run - -.. code:: console - - root@b1f6650222bc:/code# reframe -c tutorials/flux -C tutorials/flux/settings.py --run - [ReFrame Setup] - version: 4.0.0-dev.1 - command: '/code/bin/reframe -c tutorials/flux -C tutorials/flux/settings.py --run' - launched by: root@b1f6650222bc - working directory: '/code' - settings file: 'tutorials/flux/settings.py' - check search path: '/code/tutorials/flux' - stage directory: '/code/stage' - output directory: '/code/output' - - [==========] Running 1 check(s) - [==========] Started on Fri Sep 16 20:47:15 2022 - - [----------] start processing checks - [ RUN ] EchoRandTest /66b93401 @generic:default+builtin - [ OK ] (1/1) EchoRandTest /66b93401 @generic:default+builtin - [----------] all spawned checks have finished - - [ PASSED ] Ran 1/1 test case(s) from 1 check(s) (0 failure(s), 0 skipped) - [==========] Finished on Fri Sep 16 20:47:15 2022 - Run report saved in '/root/.reframe/reports/run-report.json' - Log file(s) saved in '/tmp/rfm-0avso9nb.log' - -For advanced users or developers, here is how to run tests within the container: - -Testing -------- - -.. code:: console - - ./test_reframe.py --rfm-user-config=tutorials/flux/settings.py unittests/test_schedulers.py -xs diff --git a/docs/tutorial_make_test.rst b/docs/tutorial_make_test.rst deleted file mode 100644 index 476e3502fb..0000000000 --- a/docs/tutorial_make_test.rst +++ /dev/null @@ -1,89 +0,0 @@ -Tutorial 8: Generating tests programmatically -============================================= - -You can use ReFrame to generate tests programmatically using the special :func:`~reframe.core.meta.make_test` function. -This function creates a new test type as if you have typed it manually using the :keyword:`class` keyword. -You can create arbitrarily complex tests that use variables, parameters, fixtures and pipeline hooks. - -In this tutorial, we will use :func:`~reframe.core.meta.make_test` to build a simple domain-specific syntax for generating variants of STREAM benchmarks. -Our baseline STREAM test is the following: - -.. literalinclude:: ../tutorials/advanced/make_test/stream.py - -It is essentially the STREAM benchmark split in two tests: one that builds the binaries based on set of variables and another one that runs it. - -For our example, we would like to create a simpler syntax for generating multiple different :class:`stream_test` versions that could run all at once. -Here is an example specification file for those tests: - -.. literalinclude:: ../tutorials/advanced/make_test/stream_config.yaml - -The :attr:`thread_scaling` configuration parameter for the last workflow will create a parameterised version of the test using different number of threads. -In total, we expect six :class:`stream_test` versions to be generated by this configuration. - -The process for generating the actual tests from this spec file comprises three steps and everything happens in a somewhat unconventional, though valid, ReFrame test file: - -1. We load the test configuration from a spec file that is passed through the ``STREAM_SPEC_FILE`` environment variable. -2. Based on the loaded test specs we generate the actual tests using the :func:`~reframe.core.meta.make_test` function. -3. We register the generated tests with the framework by applying manually the :func:`@simple_test ` decorator. - -The whole code for generating the tests is the following and is only a few lines. -Let's walk through it. - -.. literalinclude:: ../tutorials/advanced/make_test/stream_workflows.py - -The :func:`load_specs()` function simply loads the test specs from the YAML test spec file and does some simple sanity checking. - -The :func:`generate_tests()` function consumes the test specs and generates a test for each entry. -Each test inherits from the base :class:`stream_test` and redefines its :attr:`stream_binaries` fixture so that it is instantiated with the set of variables specified in the test spec. -Remember that all the STREAM test variables in the YAML file refer to its build phase and thus its build fixture. -We also treat specially the :attr:`thread_scaling` spec parameter. -In this case, we add a :attr:`num_threads` parameter to the test and add a post-init hook that sets the test's :attr:`~reframe.core.pipeline.RegressionTest.num_cpus_per_task`. - -Finally, we register the generated tests using the :func:`rfm.simple_test` decorator directly; -remember that :func:`~reframe.core.meta.make_test` returns a class. - -The equivalent of our test generation for the third spec is exactly the following: - -.. code-block:: python - - @rfm.simple_test - class stream_test_2(stream_test): - stream_binaries = fixture(stream_build, scope='environment', - variables={'elem_type': 'double', - 'array_size': 16777216, - 'num_iters': 10}) - num_threads = parameter([1, 2, 4, 8]) - - @run_after('init') - def _set_num_threads(self): - self.num_cpus_per_task = self.num_threads - -And here is the listing of generated tests: - -.. code-block:: bash - - STREAM_SPEC_FILE=stream_config.yaml ./bin/reframe -C tutorials/cscs-webinar-2022/config/mysettings.py -c tutorials/advanced/make_test/stream_workflows.py -l - -.. code-block:: console - - [List of matched checks] - - stream_test_2 %num_threads=8 %stream_binaries.elem_type=double %stream_binaries.array_size=16777216 %stream_binaries.num_iters=10 /7b20a90a - ^stream_build %elem_type=double %array_size=16777216 %num_iters=10 ~tresa:default+gnu 'stream_binaries /1dd920e5 - - stream_test_2 %num_threads=4 %stream_binaries.elem_type=double %stream_binaries.array_size=16777216 %stream_binaries.num_iters=10 /7cbd26d7 - ^stream_build %elem_type=double %array_size=16777216 %num_iters=10 ~tresa:default+gnu 'stream_binaries /1dd920e5 - - stream_test_2 %num_threads=2 %stream_binaries.elem_type=double %stream_binaries.array_size=16777216 %stream_binaries.num_iters=10 /797fb1ed - ^stream_build %elem_type=double %array_size=16777216 %num_iters=10 ~tresa:default+gnu 'stream_binaries /1dd920e5 - - stream_test_2 %num_threads=1 %stream_binaries.elem_type=double %stream_binaries.array_size=16777216 %stream_binaries.num_iters=10 /7a7dcd20 - ^stream_build %elem_type=double %array_size=16777216 %num_iters=10 ~tresa:default+gnu 'stream_binaries /1dd920e5 - - stream_test_1 %stream_binaries.elem_type=double %stream_binaries.array_size=1048576 %stream_binaries.num_iters=100 %stream_binaries.num_cpus_per_task=1 /3e3643dd - ^stream_build %elem_type=double %array_size=1048576 %num_iters=100 %num_cpus_per_task=1 ~tresa:default+gnu 'stream_binaries /3611a49a - - stream_test_0 %stream_binaries.elem_type=float %stream_binaries.array_size=16777216 %stream_binaries.num_iters=10 %stream_binaries.num_cpus_per_task=4 /d99b89f1 - ^stream_build %elem_type=float %array_size=16777216 %num_iters=10 %num_cpus_per_task=4 ~tresa:default+gnu 'stream_binaries /321abb06 - Found 6 check(s) - - -.. note:: - - The path passed to ``STREAM_SPEC_FILE`` is relative to the test directory. - Since version 4.2, ReFrame changes to the test directory before loading a test fil. - In prior versions you have to specify the path relative to the current working directory. diff --git a/docs/tutorial_tips_tricks.rst b/docs/tutorial_tips_tricks.rst deleted file mode 100644 index d50888b2a8..0000000000 --- a/docs/tutorial_tips_tricks.rst +++ /dev/null @@ -1,509 +0,0 @@ -=========================== -Tutorial 6: Tips and Tricks -=========================== - -.. versionadded:: 3.4 - -This tutorial focuses on some less known aspects of ReFrame's command line interface that can be helpful. - - -Debugging ---------- - -ReFrame tests are Python classes inside Python source files, so the usual debugging techniques for Python apply, but the ReFrame frontend will filter some errors and stack traces by default in order to keep the output clean. -Generally, ReFrame will not print the full stack trace for user programming errors and will not block the test loading process. -If a test has errors and cannot be loaded, an error message will be printed and the loading of the remaining tests will continue. -In the following, we have inserted a small typo in the ``hello2.py`` tutorial example: - -.. code:: bash - - ./bin/reframe -c tutorials/basics/hello -R -l - -.. literalinclude:: listings/hello2_typo.txt - :language: console - -Notice how ReFrame prints also the source code line that caused the error. -This is not always the case, however. -ReFrame cannot always track a user error back to its source and this is particularly true for the ReFrame-specific syntactic elements, such as the class `builtins `__. -In such cases, ReFrame will just print the error message but not the source code context. -In the following example, we introduce a typo in the argument of the :obj:`@run_before` decorator: - -.. code-block:: none - - ./bin/reframe: skipping test file '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py': reframe syntax error: invalid pipeline stage specified: 'compil' (rerun with '-v' for more information) - [List of matched checks] - - HelloTest (found in '/Users/user/Repositories/reframe/tutorials/basics/hello/hello1.py') - Found 1 check(s) - - -As suggested by the warning message, passing :option:`-v` will give you the stack trace for each of the failing tests, as well as some more information about what is going on during the loading. - -.. code:: bash - - ./bin/reframe -c tutorials/basics/hello -R -l -v - -.. literalinclude:: listings/hello2_typo_stacktrace.txt - :language: console - -.. tip:: - The :option:`-v` option can be given multiple times to increase the verbosity level further. - - -Debugging deferred expressions -============================== - -Although deferred expressions that are used in sanity and performance functions behave similarly to normal Python expressions, you need to understand their `implicit evaluation rules `__. -One of the rules is that :func:`str` triggers the implicit evaluation, so trying to use the standard :func:`print` function with a deferred expression, you might get unexpected results if that expression is not yet to be evaluated. -For this reason, ReFrame offers a sanity function counterpart of :func:`print`, which allows you to safely print deferred expressions. - -Let's see that in practice, by printing the filename of the standard output for :class:`HelloMultiLangTest` test. -The :attr:`stdout ` is a deferred expression and it will get its value later on while the test executes. -Trying to use the standard print here :func:`print` function here would be of little help, since it would simply give us :obj:`None`, which is the value of :attr:`stdout` when the test is created. - - -.. code-block:: python - :emphasize-lines: 15-17 - - import reframe as rfm - import reframe.utility.sanity as sn - - - @rfm.simple_test - class HelloMultiLangTest(rfm.RegressionTest): - lang = parameter(['c', 'cpp']) - valid_systems = ['*'] - valid_prog_environs = ['*'] - - @run_after('compile') - def set_sourcepath(self): - self.sourcepath = f'hello.{self.lang}' - - @sanity_function - def validate_output(self): - return sn.assert_found(r'Hello, World\!', sn.print(self.stdout)) - - -If we run the test, we can see that the correct standard output filename will be printed after sanity: - -.. code:: bash - - ./bin/reframe -C tutorials/config/settings.py -c tutorials/basics/hello/hello2.py -r - -.. literalinclude:: listings/hello2_print_stdout.txt - :language: console - - -Debugging sanity and performance patterns -========================================= -When creating a new test that requires a complex output parsing for either the ``sanity`` or ``performance`` pipeline stages, tuning the functions decorated by :attr:`@sanity_function` or :attr:`@performance_function` may involve some trial and error to debug the complex regular expressions required. -For lightweight tests which execute in a few seconds, this trial and error may not be an issue at all. -However, when dealing with tests which take longer to run, this method can quickly become tedious and inefficient. - -.. tip:: - When dealing with ``make``-based projects which take a long time to compile, you can use the command line option :option:`--dont-restage` in order to speed up the compile stage in subsequent runs. - -When a test fails, ReFrame will keep the test output in the stage directory after its execution, which means that one can load this output into a Python shell or another helper script without having to rerun the expensive test again. -If the test is not failing but the user still wants to experiment or modify the existing sanity or performance functions, the command line option :option:`--keep-stage-files` can be used when running ReFrame to avoid deleting the stage directory. -With the executable's output available in the stage directory, one can simply use the `re `_ module to debug regular expressions as shown below. - -.. code-block:: python - - >>> import re - - >>> # Read the test's output - >>> with open(the_output_file, 'r') as f: - ... test_output = ''.join(f.readlines()) - ... - >>> # Evaluate the regular expression - >>> re.findall(the_regex_pattern, test_output, re.MULTILINE) - -Alternatively to using the `re `_ module, one could use all the :mod:`~reframe.utility.sanity` utility provided by ReFrame directly from the Python shell. -In order to do so, if ReFrame was installed manually using the ``bootstrap.sh`` script, one will have to make all the Python modules from the ``external`` directory accessible to the Python shell as shown below. - -.. code-block:: python - - >>> import sys - >>> import os - - >>> # Make the external modules available - >>> sys.path = [os.path.abspath('external')] + sys.path - - >>> # Import ReFrame-provided sanity functions - >>> import reframe.utility.sanity as sn - - >>> # Evaluate the regular expression - >>> assert sn.evaluate(sn.assert_found(the_regex_pattern, the_output_file)) - - -Debugging test loading -====================== - -If you are new to ReFrame, you might wonder sometimes why your tests are not loading or why your tests are not running on the partition they were supposed to run. -This can be due to ReFrame picking the wrong configuration entry or that your test is not written properly (not decorated, no :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` etc.). -If you try to load a test file and list its tests by increasing twice the verbosity level, you will get enough output to help you debug such issues. -Let's try loading the ``tutorials/basics/hello/hello2.py`` file: - -.. code:: bash - - ./bin/reframe -C tutorials/config/settings.py -c tutorials/basics/hello/hello2.py -l -vv - -.. literalinclude:: listings/hello2_list_verbose.txt - :language: console - -You can see all the different phases ReFrame's frontend goes through when loading a test. -After loading the configuration, ReFrame will print out its relevant environment variables and will start examining the given files in order to find and load ReFrame tests. -Before attempting to load a file, it will validate it and check if it looks like a ReFrame test. -If it does, it will load that file by importing it. -This is where any ReFrame tests are instantiated and initialized (see ``Loaded 2 test(s)``), as well as the actual test cases (combination of tests, system partitions and environments) are generated. -Then the test cases are filtered based on the various `filtering command line options `__ as well as the programming environments that are defined for the currently selected system. -Finally, the test case dependency graph is built and everything is ready for running (or listing). - -Try passing a specific system or partition with the :option:`--system` option or modify the test (e.g., removing the decorator that registers it) and see how the logs change. - - -Execution modes ---------------- - -ReFrame allows you to create pre-defined ways of running it, which you can invoke from the command line. -These are called *execution modes* and are essentially named groups of command line options that will be passed to ReFrame whenever you request them. -These are defined in the configuration file and can be requested with the :option:`--mode` command-line option. -The following configuration defines an execution mode named ``maintenance`` and sets up ReFrame in a certain way (selects tests to run, sets up stage and output paths etc.) - -.. code-block:: python - - 'modes': [ - { - 'name': 'maintenance', - 'options': [ - '--unload-module=reframe', - '--exec-policy=async', - '-S strict_check=1', - '--output=/path/to/$USER/regression/maintenance', - '--perflogdir=/path/to/$USER/regression/maintenance/logs', - '--stage=$SCRATCH/regression/maintenance/stage', - '--report-file=/path/to/$USER/regression/maintenance/reports/maint_report_{sessionid}.json', - '-Jreservation=maintenance', - '--save-log-files', - '--tag=maintenance', - '--timestamp=%F_%H-%M-%S' - ] - }, - ] - -The execution modes come handy in situations that you have a standardized way of running ReFrame and you don't want to create and maintain shell scripts around it. -In this example, you can simply run ReFrame with - -.. code:: bash - - ./bin/reframe --mode=maintenance -r - -and it will be equivalent to passing explicitly all the above options. -You can still pass any additional command line option and it will supersede or be combined (depending on the behaviour of the option) with those defined in the execution mode. -In this particular example, we could change just the reservation name by running - -.. code:: bash - - ./bin/reframe --mode=maintenance -J reservation=maint -r - -There are two options that you can't use inside execution modes and these are the :option:`-C` and :option:`--system`. -The reason is that these option select the configuration file and the configuration entry to load. - - -Manipulating ReFrame's environment ----------------------------------- - -ReFrame runs the selected tests in the same environment as the one that it executes. -It does not unload any environment modules nor sets or unsets any environment variable. -Nonetheless, it gives you the opportunity to modify the environment that the tests execute. -You can either purge completely all environment modules by passing the :option:`--purge-env` option or ask ReFrame to load or unload some environment modules before starting running any tests by using the :option:`-m` and :option:`-u` options respectively. -Of course you could manage the environment manually, but it's more convenient if you do that directly through ReFrame's command-line. -If you used an environment module to load ReFrame, e.g., ``reframe``, you can use the :option:`-u` to have ReFrame unload it before running any tests, so that the tests start in a clean environment: - -.. code:: bash - - ./bin/reframe -u reframe [...] - - -Environment Modules Mappings ----------------------------- - -ReFrame allows you to replace environment modules used in tests with other modules on the fly. -This is quite useful if you want to test a new version of a module or another combination of modules. -Assume you have a test that loads a ``gromacs`` module: - -.. code-block:: python - - class GromacsTest(rfm.RunOnlyRegressionTest): - ... - modules = ['gromacs'] - - -This test would use the default version of the module in the system, but you might want to test another version, before making that new one the default. -You can ask ReFrame to temporarily replace the ``gromacs`` module with another one as follows: - - -.. code-block:: bash - - ./bin/reframe -n GromacsTest -M 'gromacs:gromacs/2020.5' -r - - -Every time ReFrame tries to load the ``gromacs`` module, it will replace it with ``gromacs/2020.5``. -You can specify multiple mappings at once or provide a file with mappings using the :option:`--module-mappings` option. -You can also replace a single module with multiple modules. - -A very convenient feature of ReFrame in dealing with modules is that you do not have to care about module conflicts at all, regardless of the modules system backend. -ReFrame will take care of unloading any conflicting modules, if the underlying modules system cannot do that automatically. -In case of module mappings, it will also respect the module order of the replacement modules and will produce the correct series of "load" and "unload" commands needed by the modules system backend used. - - -Retrying and Rerunning Tests ----------------------------- - -If you are running ReFrame regularly as part of a continuous testing procedure you might not want it to generate alerts for transient failures. -If a ReFrame test fails, you might want to retry a couple of times before marking it as a failure. -You can achieve this with the :option:`--max-retries`. -ReFrame will then retry the failing test cases a maximum number of times before reporting them as actual failures. -The failed test cases will not be retried immediately after they have failed, but rather at the end of the run session. -This is done to give more chances of success in case the failures have been transient. - -Another interesting feature introduced in ReFrame 3.4 is the ability to restore a previous test session. -Whenever it runs, ReFrame stores a detailed JSON report of the last run under ``$HOME/.reframe`` (see :option:`--report-file`). -Using that file, ReFrame can restore a previous run session using the :option:`--restore-session`. -This option is useful when you combine it with the various test filtering options. -For example, you might want to rerun only the failed tests or just a specific test in a dependency chain. -Let's see an artificial example that uses the following test dependency graph. - -.. _fig-deps-complex: - -.. figure:: _static/img/deps-complex.svg - :align: center - - :sub:`Complex test dependency graph. Nodes in red are set to fail.` - - - -Tests :class:`T2` and :class:`T8` are set to fail. -Let's run the whole test DAG: - -.. code-block:: bash - - ./bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -r - -.. literalinclude:: listings/deps_complex_run.txt - :language: console - -You can restore the run session and run only the failed test cases as follows: - - -.. code-block:: bash - - ./bin/reframe --restore-session --failed -r - - -Of course, as expected, the run will fail again, since these tests were designed to fail. - -Instead of running the failed test cases of a previous run, you might simply want to rerun a specific test. -This has little meaning if you don't use dependencies, because it would be equivalent to running it separately using the :option:`-n` option. -However, if a test was part of a dependency chain, using :option:`--restore-session` will not rerun its dependencies, but it will rather restore them. -This is useful in cases where the test that we want to rerun depends on time-consuming tests. -There is a little tweak, though, for this to work: -you need to have run with :option:`--keep-stage-files` in order to keep the stage directory even for tests that have passed. -This is due to two reasons: -(a) if a test needs resources from its parents, it will look into their stage directories and -(b) ReFrame stores the state of a finished test case inside its stage directory and it will need that state information in order to restore a test case. - -Let's try to rerun the :class:`T6` test from the previous test dependency chain: - - -.. code-block:: bash - - ./bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py --keep-stage-files -r - -.. code-block:: bash - - ./bin/reframe --restore-session --keep-stage-files -n T6 -r - - -Notice how only the :class:`T6` test was rerun and none of its dependencies, since they were simply restored: - -.. literalinclude:: listings/deps_rerun_t6.txt - :language: console - - -If we tried to run :class:`T6` without restoring the session, we would have to rerun also the whole dependency chain, i.e., also :class:`T5`, :class:`T1`, :class:`T4` and :class:`T0`. - -.. code-block:: bash - - ./bin/reframe -c unittests/resources/checks_unlisted/deps_complex.py -n T6 -r - -.. literalinclude:: listings/deps_run_t6.txt - :language: console - - -Implementing test workarounds efficiently ------------------------------------------ - -.. versionadded:: 3.2 - -Sometimes you may need to add a quick workaround in a test, because something in a system or an environment broken. -The best way to implement this is through hooks, because you can easily disable any hook from the command-line and you don't need to update the test every time you want to check if the system is fixed and the workaround is not needed anymore. - -Let's use one example from the `previous tutorial `__ and let's assume that there is something wrong with one of the environments and a special macro needs to be defined in order for the compilation to succeed. -Instead of adding another flag in the :func:`set_compilation_flags` hook, it is better to add another hook containing just the workaround as shown below: - -.. code-block:: python - :emphasize-lines: 27-33 - - import reframe as rfm - import reframe.utility.sanity as sn - - - @rfm.simple_test - class HelloThreadedExtended2Test(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - sourcepath = 'hello_threads.cpp' - build_system = 'SingleSource' - executable_opts = ['16'] - - @run_before('compile') - def set_compilation_flags(self): - self.build_system.cppflags = ['-DSYNC_MESSAGES'] - self.build_system.cxxflags = ['-std=c++11', '-Wall'] - environ = self.current_environ.name - if environ in {'clang', 'gnu'}: - self.build_system.cxxflags += ['-pthread'] - - @sanity_function - def assert_num_messages(self): - num_messages = sn.len(sn.findall(r'\[\s?\d+\] Hello, World\!', - self.stdout)) - return sn.assert_eq(num_messages, 16) - - @run_before('compile') - def fooenv_workaround(self): - ce = self.current_environ.name - if ce == 'foo': - self.build_system.cppflags += [ - '-D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL' - ] - -This way the test will start passing again allowing us to catch any new issues while waiting for the original issue to be fixed. -Then we can run the test anytime using ``--disable-hook=fooenv_workaround`` to check if the workaround is not needed anymore. - - -Import user modules from a test file ------------------------------------- - -When building complex test suites or test libraries it is often the case that you would like to abstract away common functionality in a different Python module and import when needed. -Suppose the following test directory structure: - -.. code-block:: console - - tutorials/advanced/user_imports/ - ├── commonutil - │   ├── __init__.py - └── tests - ├── test.py - └── testutil.py - -The ``commonutil`` module defines a :func:`greetings` function and the ``testutil`` module defines the :func:`greetings_from_test`. -Suppose that the tests defined in ``test.py`` would like to use both of these modules. -Prior to ReFrame 4.2, users would have to explicitly modify the :obj:`sys.path` in their test code before importing the desired modules as follows: - -.. code-block:: python - - import os - import sys - - prefix = os.path.dirname(__file__) - sys.path += [os.path.join(prefix, '..'), prefix] - - import commonutil - from testutil import greetings_from_test - -This is a lot of boilerplate code for a relatively common operation. -ReFrame 4.2 improves substantially this by introducing the following changes: - -1. The directory of the test file is temporarily added to the :obj:`sys.path` when ReFrame is loading the test file, so the ``testutil`` module can be directly imported without any additional preparation. -2. While loading the test, ReFrame changes to the test's directory. -3. ReFrame 4.2 provides two new utility functions for importing modules or symbols from modules: the :func:`~reframe.utility.import_module` and the :func:`~reframe.utility.import_from_module` - -The last two modifications allow users to load a module that is not in the same directory as the test file, like the ``commonutil`` module in this example. -Let's rewrite the previous imports in ReFrame 4.2: - -.. code-block:: - - import reframe.utility as util - from testutil import greetings_from_test - - commonutil = util.import_module('..commonutil') - - -As soon as ``commonutil`` is imported with the utility function, it can be used as if it has been imported with an ``import`` statement. - -.. note:: - - Python will complain if you try to ``import ..commonutil`` as the test file is not part of a parent package. - - -.. _generate-ci-pipeline: - -Integrating into a CI pipeline ------------------------------- - -.. versionadded:: 3.4.1 - -Instead of running your tests, you can ask ReFrame to generate a `child pipeline `__ specification for the Gitlab CI. -This will spawn a CI job for each ReFrame test respecting test dependencies. -You could run your tests in a single job of your Gitlab pipeline, but you would not take advantage of the parallelism across different CI jobs. -Having a separate CI job per test makes it also easier to spot the failing tests. - -As soon as you have set up a `runner `__ for your repository, it is fairly straightforward to use ReFrame to automatically generate the necessary CI steps. -The following is an example of ``.gitlab-ci.yml`` file that does exactly that: - -.. code-block:: yaml - - stages: - - generate - - test - - generate-pipeline: - stage: generate - script: - - reframe --ci-generate=${CI_PROJECT_DIR}/pipeline.yml -c ${CI_PROJECT_DIR}/path/to/tests - artifacts: - paths: - - ${CI_PROJECT_DIR}/pipeline.yml - - test-jobs: - stage: test - trigger: - include: - - artifact: pipeline.yml - job: generate-pipeline - strategy: depend - - -It defines two stages. -The first one, called ``generate``, will call ReFrame to generate the pipeline specification for the desired tests. -All the usual `test selection options `__ can be used to select specific tests. -ReFrame will process them as usual, but instead of running the selected tests, it will generate the correct steps for running each test individually as a Gitlab job in a child pipeline. -The generated ReFrame command that will run each individual test reuses the :option:`-C`, :option:`-R`, :option:`-v` and :option:`--mode` options passed to the initial invocation of ReFrame that was used to generate the pipeline. -Users can define CI-specific execution modes in their configuration in order to pass arbitrary options to the ReFrame invocation in the child pipeline. - -Finally, we pass the generated CI pipeline file to second phase as an artifact and we are done! -If ``image`` keyword is defined in ``.gitlab-ci.yml``, the emitted pipeline will use the same image as the one defined in the parent pipeline. -Besides, each job in the generated pipeline will output a separate junit report which can be used to create GitLab badges. - -The following figure shows one part of the automatically generated pipeline for the test graph depicted `above <#fig-deps-complex>`__. - -.. figure:: _static/img/gitlab-ci.png - :align: center - - :sub:`Snapshot of a Gitlab pipeline generated automatically by ReFrame.` - - -.. note:: - - The ReFrame executable must be available in the Gitlab runner that will run the CI jobs. diff --git a/docs/tutorials.rst b/docs/tutorials.rst index ecee26e7f8..c7c6426f50 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -1,15 +1,8 @@ -================= +:orphan: + ReFrame Tutorials ================= - -.. toctree:: - - tutorial_basics - tutorial_advanced - tutorial_deps - tutorial_fixtures - tutorial_build_automation - tutorial_tips_tricks - tutorial_flux - tutorial_make_test +.. attention:: + This page has been replaced. + Check out the new :doc:`tutorial ` and the :doc:`howto ` pages. diff --git a/docs/whats_new_40.rst b/docs/whats_new_40.rst index e5780479c7..e286cdd7dd 100644 --- a/docs/whats_new_40.rst +++ b/docs/whats_new_40.rst @@ -34,7 +34,7 @@ If you want to eliminate them, though, you should remove the conflicting definit Although ReFrame will not warn you for redefining other configuration sections, you are also advised to tidy up your configuration file and remove any parts that were copied unchanged from the builtin configuration. -For more information on how ReFrame 4.0 builds and loads its configuration, please refer to the documentation of the :option:`--config-file` option, as well as the :ref:`building-the-final-config` section. +For more information on how ReFrame 4.0 builds and loads its configuration, please refer to the documentation of the :option:`--config-file` option. Performance Reporting and Logging @@ -77,14 +77,14 @@ Check the documentation of the :option:`-n` for more information. For the details of the new naming scheme, please refer to the :ref:`test_naming_scheme` section. Note that any tests that used the old naming scheme to depend on parameterized tests will break with this change. -Check the tutorial :ref:`param_deps` on how to create dependencies on parameterized tests in a portable way. +Check the article :ref:`param_deps` on how to create dependencies on parameterized tests in a portable way. Custom parallel launchers ^^^^^^^^^^^^^^^^^^^^^^^^^ By relaxing the configuration schema, users can now define custom parallel launchers inside their Python configuration file. -Check the tutorial :ref:`custom_launchers` to find out how this can be achieved. +Check the article :ref:`custom-launchers` to find out how this can be achieved. Unique run reports @@ -132,4 +132,4 @@ New Deprecations - All occurrences of the ``variables`` name are deprecated in favor of ``env_vars``. This includes the :attr:`~reframe.core.pipeline.RegressionTest.variables` test attribute and the homonym systems, partitions and environments configuration parameters as well as the :attr:`~reframe.core.environments.Environment.variables` of the :attr:`~reframe.core.environments.Environment` base class. - Although :attr:`~reframe.core.pipeline.RegressionTest.perf_patterns` attribute is not deprecated, users are recommended to migrate to using the new :attr:`@performance_function ` builtin. - Please refer to :ref:`perftest-basics` tutorial for a starting point. + Please refer to :ref:`writing-your-first-test` of the tutorial for a starting point. diff --git a/tutorials/cscs-webinar-2022/config/mysettings.py b/examples/cscs-webinar-2022/config/mysettings.py similarity index 100% rename from tutorials/cscs-webinar-2022/config/mysettings.py rename to examples/cscs-webinar-2022/config/mysettings.py diff --git a/tutorials/cscs-webinar-2022/cscs-webinar-2022.cast b/examples/cscs-webinar-2022/cscs-webinar-2022.cast similarity index 100% rename from tutorials/cscs-webinar-2022/cscs-webinar-2022.cast rename to examples/cscs-webinar-2022/cscs-webinar-2022.cast diff --git a/tutorials/advanced/make_test/src/stream.c b/examples/cscs-webinar-2022/tests/src/stream.c similarity index 100% rename from tutorials/advanced/make_test/src/stream.c rename to examples/cscs-webinar-2022/tests/src/stream.c diff --git a/tutorials/cscs-webinar-2022/tests/stream1.py b/examples/cscs-webinar-2022/tests/stream1.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream1.py rename to examples/cscs-webinar-2022/tests/stream1.py diff --git a/tutorials/cscs-webinar-2022/tests/stream2.py b/examples/cscs-webinar-2022/tests/stream2.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream2.py rename to examples/cscs-webinar-2022/tests/stream2.py diff --git a/tutorials/cscs-webinar-2022/tests/stream3.py b/examples/cscs-webinar-2022/tests/stream3.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream3.py rename to examples/cscs-webinar-2022/tests/stream3.py diff --git a/tutorials/cscs-webinar-2022/tests/stream4.py b/examples/cscs-webinar-2022/tests/stream4.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream4.py rename to examples/cscs-webinar-2022/tests/stream4.py diff --git a/tutorials/cscs-webinar-2022/tests/stream5.py b/examples/cscs-webinar-2022/tests/stream5.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream5.py rename to examples/cscs-webinar-2022/tests/stream5.py diff --git a/tutorials/cscs-webinar-2022/tests/stream6.py b/examples/cscs-webinar-2022/tests/stream6.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream6.py rename to examples/cscs-webinar-2022/tests/stream6.py diff --git a/tutorials/cscs-webinar-2022/tests/stream7.py b/examples/cscs-webinar-2022/tests/stream7.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream7.py rename to examples/cscs-webinar-2022/tests/stream7.py diff --git a/tutorials/cscs-webinar-2022/tests/stream8.py b/examples/cscs-webinar-2022/tests/stream8.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream8.py rename to examples/cscs-webinar-2022/tests/stream8.py diff --git a/tutorials/cscs-webinar-2022/tests/stream9.py b/examples/cscs-webinar-2022/tests/stream9.py similarity index 100% rename from tutorials/cscs-webinar-2022/tests/stream9.py rename to examples/cscs-webinar-2022/tests/stream9.py diff --git a/tutorials/flux/README.md b/examples/howto/flux/README.md similarity index 100% rename from tutorials/flux/README.md rename to examples/howto/flux/README.md diff --git a/tutorials/flux/example1.py b/examples/howto/flux/example1.py similarity index 100% rename from tutorials/flux/example1.py rename to examples/howto/flux/example1.py diff --git a/tutorials/flux/settings.py b/examples/howto/flux/settings.py similarity index 100% rename from tutorials/flux/settings.py rename to examples/howto/flux/settings.py diff --git a/examples/howto/testlib/__init__.py b/examples/howto/testlib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/howto/testlib/simple.py b/examples/howto/testlib/simple.py new file mode 100644 index 0000000000..1316168bd5 --- /dev/null +++ b/examples/howto/testlib/simple.py @@ -0,0 +1,27 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn +from .utility import dummy_fixture + + +@rfm.simple_test +class simple_echo_check(rfm.RunOnlyRegressionTest, pin_prefix=True): + descr = 'Simple Echo Test' + valid_systems = ['*'] + valid_prog_environs = ['builtin'] + executable = 'echo' + executable_opts = ['Hello'] + message = variable(str, value='World') + dummy = fixture(dummy_fixture, scope='environment') + + @run_before('run') + def set_executable_opts(self): + self.executable_opts += [self.message] + + @sanity_function + def assert_sanity(self): + return sn.assert_found(rf'Hello {self.message}', self.stdout) diff --git a/examples/howto/testlib/utility/__init__.py b/examples/howto/testlib/utility/__init__.py new file mode 100644 index 0000000000..72f9a7f3a2 --- /dev/null +++ b/examples/howto/testlib/utility/__init__.py @@ -0,0 +1,7 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +class dummy_fixture(rfm.RunOnlyRegressionTest): + executable = 'echo' + sanity_patterns = sn.assert_true(1) diff --git a/examples/howto/testlib_example.py b/examples/howto/testlib_example.py new file mode 100644 index 0000000000..ad59d4ff41 --- /dev/null +++ b/examples/howto/testlib_example.py @@ -0,0 +1,12 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +from testlib.simple import simple_echo_check + + +@rfm.simple_test +class HelloFoo(simple_echo_check): + message = 'Foo' diff --git a/examples/tutorial/config/baseline.py b/examples/tutorial/config/baseline.py new file mode 100644 index 0000000000..e7150bfdf7 --- /dev/null +++ b/examples/tutorial/config/baseline.py @@ -0,0 +1,29 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + } + ] +} diff --git a/examples/tutorial/config/baseline_contplatf.py b/examples/tutorial/config/baseline_contplatf.py new file mode 100644 index 0000000000..d0fef93ff8 --- /dev/null +++ b/examples/tutorial/config/baseline_contplatf.py @@ -0,0 +1,24 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['.*'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['builtin'], + 'container_platforms': [{'type': 'Docker'}] + } + ] + } + ] +} diff --git a/examples/tutorial/config/baseline_environs.py b/examples/tutorial/config/baseline_environs.py new file mode 100644 index 0000000000..6f17433879 --- /dev/null +++ b/examples/tutorial/config/baseline_environs.py @@ -0,0 +1,43 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ] +} diff --git a/tutorials/config/lmodsys.py b/examples/tutorial/config/baseline_modules.py similarity index 66% rename from tutorials/config/lmodsys.py rename to examples/tutorial/config/baseline_modules.py index bf12d8258f..f6e666660a 100644 --- a/tutorials/config/lmodsys.py +++ b/examples/tutorial/config/baseline_modules.py @@ -1,19 +1,19 @@ # Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# and other ReFrame Project Developers. See the top-level LICENSE file for -# details. +# ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause site_configuration = { 'systems': [ { - 'name': 'lmodsys', - 'descr': 'Generic system with Lmod', - 'hostnames': ['.*'], + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], 'modules_system': 'lmod', 'partitions': [ { 'name': 'default', + 'descr': 'Example partition', 'scheduler': 'local', 'launcher': 'local', 'environs': ['builtin'] diff --git a/examples/tutorial/config/cluster.py b/examples/tutorial/config/cluster.py new file mode 100644 index 0000000000..debb399b40 --- /dev/null +++ b/examples/tutorial/config/cluster.py @@ -0,0 +1,71 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + }, + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ], + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ] +} diff --git a/examples/tutorial/config/cluster_logging.py b/examples/tutorial/config/cluster_logging.py new file mode 100644 index 0000000000..02b05662a8 --- /dev/null +++ b/examples/tutorial/config/cluster_logging.py @@ -0,0 +1,94 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + }, + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ], + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ], + 'logging': [ + { + 'handlers': [ + { + 'type': 'file', + 'name': 'reframe.log', + 'timestamp': '%FT%T', + 'level': 'debug2', + 'format': ('[%(asctime)s] %(levelname)s: ' + '%(check_info)s: %(message)s'), + 'append': False + }, + { + 'type': 'file', + 'name': 'reframe.out', + 'timestamp': '%FT%T', + 'level': 'info', + 'format': '%(message)s', + 'append': False + } + ] + } + ] +} diff --git a/examples/tutorial/config/cluster_mpi.py b/examples/tutorial/config/cluster_mpi.py new file mode 100644 index 0000000000..6cae4f8e0f --- /dev/null +++ b/examples/tutorial/config/cluster_mpi.py @@ -0,0 +1,78 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + }, + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'gnu-mpi', 'clang'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'gnu-mpi', + 'cc': 'mpicc', + 'cxx': 'mpic++', + 'features': ['openmp', 'mpi'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ], + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ] +} diff --git a/examples/tutorial/config/cluster_perflogs.py b/examples/tutorial/config/cluster_perflogs.py new file mode 100644 index 0000000000..2feaefdf04 --- /dev/null +++ b/examples/tutorial/config/cluster_perflogs.py @@ -0,0 +1,90 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + }, + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ], + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ], + 'logging': [ + { + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ('%(check_result)s,' + '%(check_job_completion_time)s,' + '%(check_system)s:%(check_partition)s,' + '%(check_environ)s,' + '%(check_perfvalues)s'), + 'format_perfvars': ('%(check_perf_value)s,' + '%(check_perf_unit)s,'), + 'append': True + } + ] + } + ] +} diff --git a/examples/tutorial/config/cluster_perflogs_httpjson.py b/examples/tutorial/config/cluster_perflogs_httpjson.py new file mode 100644 index 0000000000..c0725f6cad --- /dev/null +++ b/examples/tutorial/config/cluster_perflogs_httpjson.py @@ -0,0 +1,123 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +import os + + +def _format_record(record, extras, ignore_keys): + data = {} + for attr, val in record.__dict__.items(): + if attr in ignore_keys or attr.startswith('_'): + continue + + if attr.startswith('check_'): + data[attr[6:]] = val + else: + data[attr] = val + + data.update(extras) + return json.dumps(data) + + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + }, + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ], + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ], + 'logging': [ + { + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ('%(check_result)s,' + '%(check_job_completion_time)s,' + '%(check_system)s:%(check_partition)s,' + '%(check_environ)s,' + '%(check_perfvalues)s'), + 'format_perfvars': ('%(check_perf_value)s,' + '%(check_perf_unit)s,'), + 'append': True + }, + { + 'type': 'httpjson', + 'url': 'https://httpjson-server:12345/rfm', + 'level': 'info', + 'debug': True, + 'extra_headers': {'Authorization': 'Token YOUR_API_TOKEN'}, + 'extras': { + 'facility': 'reframe', + 'data-version': '1.0' + }, + 'ignore_keys': ['check_perfvalues'], + 'json_formatter': (_format_record + if os.getenv('CUSTOM_JSON') else None) + } + ] + } + ] +} diff --git a/examples/tutorial/config/cluster_resources.py b/examples/tutorial/config/cluster_resources.py new file mode 100644 index 0000000000..9e0b850606 --- /dev/null +++ b/examples/tutorial/config/cluster_resources.py @@ -0,0 +1,71 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'tutorialsys', + 'descr': 'Example system', + 'hostnames': ['myhost'], + 'partitions': [ + { + 'name': 'default', + 'descr': 'Example partition', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['baseline', 'gnu', 'clang'] + } + ] + }, + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'], + 'resources': [ + { + 'name': 'memory', + 'options': ['--mem={size}'] + } + ] + } + ] + } + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ] +} diff --git a/examples/tutorial/config/multifile/common/settings.py b/examples/tutorial/config/multifile/common/settings.py new file mode 100644 index 0000000000..71896c8391 --- /dev/null +++ b/examples/tutorial/config/multifile/common/settings.py @@ -0,0 +1,32 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ], + 'logging': [ + { + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ('%(check_result)s,' + '%(check_job_completion_time)s,' + '%(check_system)s:%(check_partition)s,' + '%(check_environ)s,' + '%(check_perfvalues)s'), + 'format_perfvars': ('%(check_perf_value)s,' + '%(check_perf_unit)s,'), + 'append': True + } + ] + } + ] +} diff --git a/examples/tutorial/config/multifile/environments/settings.py b/examples/tutorial/config/multifile/environments/settings.py new file mode 100644 index 0000000000..f891fe4205 --- /dev/null +++ b/examples/tutorial/config/multifile/environments/settings.py @@ -0,0 +1,27 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ] +} diff --git a/examples/tutorial/config/multifile/pseudo-cluster/settings.py b/examples/tutorial/config/multifile/pseudo-cluster/settings.py new file mode 100644 index 0000000000..8416e92c5f --- /dev/null +++ b/examples/tutorial/config/multifile/pseudo-cluster/settings.py @@ -0,0 +1,31 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'pseudo-cluster', + 'descr': 'Example Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute', + 'descr': 'Compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + } + ] + } + ] +} diff --git a/tutorials/advanced/containers/container_test.py b/examples/tutorial/containers/container_test.py similarity index 63% rename from tutorials/advanced/containers/container_test.py rename to examples/tutorial/containers/container_test.py index f3a413cfa1..1fe1072f96 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/examples/tutorial/containers/container_test.py @@ -9,16 +9,13 @@ @rfm.simple_test class ContainerTest(rfm.RunOnlyRegressionTest): - platform = parameter(['Sarus', 'Singularity']) - valid_systems = ['daint:gpu'] - valid_prog_environs = ['builtin'] + container_platform = 'Docker' + valid_systems = ['*'] + valid_prog_environs = ['*'] @run_before('run') def setup_container_platf(self): - self.descr = f'Run commands inside a container using {self.platform}' - image_prefix = 'docker://' if self.platform == 'Singularity' else '' - self.container_platform = self.platform - self.container_platform.image = f'{image_prefix}ubuntu:18.04' + self.container_platform.image = f'ubuntu:18.04' self.container_platform.command = ( "bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'" ) diff --git a/examples/tutorial/deps/deps_complex.py b/examples/tutorial/deps/deps_complex.py new file mode 100644 index 0000000000..64302ee27e --- /dev/null +++ b/examples/tutorial/deps/deps_complex.py @@ -0,0 +1,197 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import os +import reframe as rfm +import reframe.utility.sanity as sn +import reframe.utility.typecheck as typ + +# +# The following tests implement the dependency graph below: +# +# +# t0 +# ^ +# | +# +-->t4<--+ +# | | +# t5<------t1<--t8<--t9 +# ^ ^ +# | | +# +---t6---+ +# ^ +# | +# +<--t2<--t7 +# ^ +# | +# t3 +# +# +# Each test has an id, which is the digit in its name and it produces its +# output in the 'out.txt' file. Each test sums up its own id with the output +# produced by its parents and writes in its output file. +# + + +class BaseTest(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + sourcesdir = None + executable = 'echo' + keep_files = ['out.txt'] + count = variable(int) + deps = variable(typ.List[str], value=[]) + + @run_after('init') + def init_deps(self): + self.count = int(self.unique_name[1:]) + for d in self.deps: + self.depends_on(d) + + @run_before('run') + def write_count(self): + self.executable_opts = [str(self.count), '> out.txt'] + + +# NOTE: The order of the tests here should not be topologically sorted + + +@rfm.simple_test +class T0(BaseTest): + sanity_patterns = sn.assert_true(1) + + +@rfm.simple_test +class T1(BaseTest): + deps = ['T4', 'T5'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 14) + + @require_deps + def prepend_output(self, T4, T5): + with open(os.path.join(T4().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + with open(os.path.join(T5().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + +@rfm.simple_test +class T2(BaseTest): + deps = ['T6'] + + @sanity_function + def validate(self): + # Make this test fail on purpose: expected value is 31 normally + return sn.assert_eq(self.count, 30) + + @require_deps + def prepend_output(self, T6): + with open(os.path.join(T6().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + +@rfm.simple_test +class T3(T2): + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 32) + + +@rfm.simple_test +class T4(BaseTest): + deps = ['T0'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 4) + + @require_deps + def prepend_output(self, T0): + with open(os.path.join(T0().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + +@rfm.simple_test +class T5(BaseTest): + deps = ['T4'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 9) + + @require_deps + def prepend_output(self, T4): + with open(os.path.join(T4().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + +@rfm.simple_test +class T6(BaseTest): + deps = ['T1', 'T5'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 29) + + @require_deps + def prepend_output(self, T1, T5): + with open(os.path.join(T1().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + with open(os.path.join(T5().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + +@rfm.simple_test +class T7(BaseTest): + deps = ['T2'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 38) + + @require_deps + def prepend_output(self, T2): + with open(os.path.join(T2().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + +@rfm.simple_test +class T8(BaseTest): + deps = ['T1'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 22) + + @require_deps + def prepend_output(self, T1): + with open(os.path.join(T1().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) + + @run_after('setup') + def fail(self): + # Make this test fail on purpose + raise Exception + + +@rfm.simple_test +class T9(BaseTest): + # This tests fails because of T8. It is added to make sure that + # all tests are accounted for in the summary. + + deps = ['T8'] + + @sanity_function + def validate(self): + return sn.assert_eq(self.count, 31) + + @require_deps + def prepend_output(self, T8): + with open(os.path.join(T8().stagedir, 'out.txt')) as fp: + self.count += int(fp.read()) diff --git a/tutorials/deps/parameterized.py b/examples/tutorial/deps/parameterized.py similarity index 100% rename from tutorials/deps/parameterized.py rename to examples/tutorial/deps/parameterized.py diff --git a/examples/tutorial/dockerfiles/eb-spack.dockerfile b/examples/tutorial/dockerfiles/eb-spack.dockerfile new file mode 100644 index 0000000000..645507a887 --- /dev/null +++ b/examples/tutorial/dockerfiles/eb-spack.dockerfile @@ -0,0 +1,39 @@ +# +# Execute this from the top-level ReFrame source directory +# + + +FROM ghcr.io/reframe-hpc/lmod:8.4.12 + +ENV _SPACK_VER=0.16 +ENV _EB_VER=4.4.1 + +RUN apt-get -y update && \ + apt-get -y install curl && \ + apt-get -y install sudo && \ + apt-get -y install python3-pip && \ + apt-get -y install gcc git jq libomp-dev tree vim + +# Install reframe +ARG REFRAME_TAG=develop +WORKDIR /usr/local/share +RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/reframe-hpc/reframe.git && \ + cd reframe/ && ./bootstrap.sh +ENV PATH=/usr/local/share/reframe/bin:$PATH + +# Install EasyBuild +RUN pip3 install easybuild==${_EB_VER} + +# Add tutorial user +RUN useradd -ms /bin/bash -G sudo user && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER user +WORKDIR /home/user + +# Install Spack +RUN mkdir .local && cd .local && \ + git clone --branch releases/v${_SPACK_VER} --depth 1 https://github.com/spack/spack + +RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/user/.local/spack/share/spack/setup-env.sh' > /home/user/.profile +ENV BASH_ENV /home/user/.profile diff --git a/tutorials/flux/Dockerfile b/examples/tutorial/dockerfiles/flux.dockerfile similarity index 100% rename from tutorials/flux/Dockerfile rename to examples/tutorial/dockerfiles/flux.dockerfile diff --git a/examples/tutorial/dockerfiles/singlenode.Dockerfile b/examples/tutorial/dockerfiles/singlenode.Dockerfile new file mode 100644 index 0000000000..647f26d231 --- /dev/null +++ b/examples/tutorial/dockerfiles/singlenode.Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:22.04 + +RUN apt-get -y update && \ + apt-get -y install curl && \ + apt-get -y install sudo && \ + apt-get -y install python3-pip && \ + apt-get -y install clang gcc git jq libomp-dev tree vim + +# Install reframe +ARG REFRAME_TAG=develop +ARG REFRAME_REPO=reframe-hpc +WORKDIR /usr/local/share +RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ + cd reframe/ && ./bootstrap.sh +ENV PATH=/usr/local/share/reframe/bin:$PATH + +# Install stream +RUN mkdir -p stream/bin && \ + cd stream && \ + curl -fsSLJO https://www.cs.virginia.edu/stream/FTP/Code/stream.c && \ + gcc -DSTREAM_ARRAY_SIZE=100000000 -O3 -Wall -fopenmp -o bin/stream.x stream.c +ENV PATH=/usr/local/share/stream/bin:$PATH + +# Add tutorial user +RUN useradd -ms /bin/bash -G sudo user && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# COPY examples /home/user/reframe-examples +# RUN chown -R user:user /home/user/reframe-examples +WORKDIR /home/user + +USER user diff --git a/examples/tutorial/dockerfiles/slurm-cluster/README.txt b/examples/tutorial/dockerfiles/slurm-cluster/README.txt new file mode 100644 index 0000000000..20356456e2 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/README.txt @@ -0,0 +1,6 @@ +This Slurm Docker Compose is based on https://github.com/rancavil/slurm-cluster created by: + Rodrigo Ancavil del Pino. + +Modified and adapted for this tutorial by: + Theofilos Manitaras and + Vasileios Karakasis diff --git a/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml b/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml new file mode 100644 index 0000000000..13ce87e9ff --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml @@ -0,0 +1,113 @@ +services: + munge-key-generator: + image: ghcr.io/reframe-hpc/munge-ubuntu:20.04 + hostname: munge-host + healthcheck: + test: ["CMD-SHELL", "test -f /scratch/munge.key"] + interval: 10s + timeout: 10s + retries: 5 + volumes: + - shared-scratch:/scratch + + frontend: + image: slurm-reframe + build: examples/tutorial/dockerfiles/slurm-cluster/reframe + container_name: frontend + hostname: login + user: admin + volumes: + - shared-home:/home/admin:rw + - shared-scratch:/scratch:rw + - type: bind + source: ./examples + target: /home/admin/reframe-examples + links: + - slurm-master + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + node0: + condition: service_started + node1: + condition: service_started + node2: + condition: service_started + environment: + - SLURM_CPUS_ON_NODE=1 + + slurm-master: + image: ghcr.io/reframe-hpc/slurm-master-ubuntu:20.04 + hostname: slurm-master + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + depends_on: + munge-key-generator: + condition: service_healthy + environment: + - SLURM_CPUS_ON_NODE=1 + + node0: + image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04 + hostname: nid00 + container_name: slurm-node0 + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + environment: + - SLURM_NODENAME=nid00 + - SLURM_CPUS_ON_NODE=1 + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + links: + - slurm-master + + node1: + image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04 + hostname: nid01 + container_name: slurm-node1 + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + environment: + - SLURM_NODENAME=nid01 + - SLURM_CPUS_ON_NODE=1 + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + links: + - slurm-master + + node2: + image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04 + hostname: nid02 + container_name: slurm-node2 + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + environment: + - SLURM_NODENAME=nid02 + - SLURM_CPUS_ON_NODE=1 + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + links: + - slurm-master + +volumes: + shared-home: + shared-scratch: diff --git a/examples/tutorial/dockerfiles/slurm-cluster/master/Dockerfile b/examples/tutorial/dockerfiles/slurm-cluster/master/Dockerfile new file mode 100644 index 0000000000..972837b628 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/master/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && apt install -y \ + build-essential \ + git \ + munge \ + slurmctld \ + slurm-wlm-torque \ + sudo \ + wget \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \ + echo "admin:admin" | chpasswd && \ + adduser admin sudo && \ + echo "admin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +COPY slurm.conf /etc/slurm-llnl/ +COPY cgroup.conf /etc/slurm-llnl/ +COPY docker-entrypoint.sh /etc/slurm-llnl/ + +RUN chmod +rx /etc/slurm-llnl/docker-entrypoint.sh + +RUN mkdir /scratch && \ + chown -R admin:admin /scratch + +EXPOSE 6817 6818 6819 3306 + +WORKDIR /home/admin + +ENTRYPOINT ["/etc/slurm-llnl/docker-entrypoint.sh"] diff --git a/examples/tutorial/dockerfiles/slurm-cluster/master/cgroup.conf b/examples/tutorial/dockerfiles/slurm-cluster/master/cgroup.conf new file mode 100644 index 0000000000..ba1dbdd95c --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/master/cgroup.conf @@ -0,0 +1,5 @@ +CgroupAutomount=yes +CgroupReleaseAgentDir="/etc/slurm/cgroup" +ConstrainCores=yes +ConstrainDevices=yes +ConstrainRAMSpace=yes diff --git a/examples/tutorial/dockerfiles/slurm-cluster/master/docker-entrypoint.sh b/examples/tutorial/dockerfiles/slurm-cluster/master/docker-entrypoint.sh new file mode 100755 index 0000000000..8c83e53cf8 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/master/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export SLURM_CPUS_ON_NODE=$(cat /proc/cpuinfo | grep processor | wc -l) +sudo sed -i "s/REPLACE_IT/CPUs=${SLURM_CPUS_ON_NODE}/g" /etc/slurm-llnl/slurm.conf + +sudo cp /scratch/munge.key /etc/munge/munge.key +sudo service munge start +sudo service slurmctld start + +tail -f /dev/null diff --git a/examples/tutorial/dockerfiles/slurm-cluster/master/slurm.conf b/examples/tutorial/dockerfiles/slurm-cluster/master/slurm.conf new file mode 100644 index 0000000000..9e2c782156 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/master/slurm.conf @@ -0,0 +1,153 @@ +# slurm.conf file generated by configurator.html. +# Put this file on all nodes of your cluster. +# See the slurm.conf man page for more information. +# +SlurmctldHost=slurm-master +# +#DisableRootJobs=NO +#EnforcePartLimits=NO +#Epilog= +#EpilogSlurmctld= +#FirstJobId=1 +#MaxJobId=999999 +#GresTypes= +#GroupUpdateForce=0 +#GroupUpdateTime=600 +#JobFileAppend=0 +#JobRequeue=1 +#JobSubmitPlugins=1 +#KillOnBadExit=0 +#LaunchType=launch/slurm +#Licenses=foo*4,bar +#MailProg=/bin/mail +#MaxJobCount=5000 +#MaxStepCount=40000 +#MaxTasksPerNode=128 +MpiDefault=pmi2 +#MpiParams=ports=#-# +#PluginDir= +#PlugStackConfig= +#PrivateData=jobs +#ProctrackType=proctrack/cgroup +ProctrackType=proctrack/linuxproc +#Prolog= +#PrologFlags= +#PrologSlurmctld= +#PropagatePrioProcess=0 +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +#RebootProgram= +ReturnToService=1 +#SallocDefaultCommand= +SlurmctldPidFile=/var/run/slurmctld.pid +SlurmctldPort=6817 +SlurmdPidFile=/var/run/slurmd.pid +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=root +#SlurmdUser=root +#SrunEpilog= +#SrunProlog= +StateSaveLocation=/var/spool +SwitchType=switch/none +#TaskEpilog= +TaskPlugin=task/affinity +TaskPluginParam=Sched +#TaskProlog= +#TopologyPlugin=topology/tree +#TmpFS=/tmp +#TrackWCKey=no +#TreeWidth= +#UnkillableStepProgram= +#UsePAM=0 +# +# +# TIMERS +#BatchStartTimeout=10 +#CompleteWait=0 +#EpilogMsgTime=2000 +#GetEnvTimeout=2 +#HealthCheckInterval=0 +#HealthCheckProgram= +InactiveLimit=0 +KillWait=30 +#MessageTimeout=10 +#ResvOverRun=0 +MinJobAge=300 +#OverTimeLimit=0 +SlurmctldTimeout=120 +SlurmdTimeout=300 +#UnkillableStepTimeout=60 +#VSizeFactor=0 +Waittime=0 +# +# +# SCHEDULING +#DefMemPerCPU=0 +#MaxMemPerCPU=0 +#SchedulerTimeSlice=30 +SchedulerType=sched/backfill +SelectType=select/cons_res +SelectTypeParameters=CR_Core +# +# +# JOB PRIORITY +#PriorityFlags= +#PriorityType=priority/basic +#PriorityDecayHalfLife= +#PriorityCalcPeriod= +#PriorityFavorSmall= +#PriorityMaxAge= +#PriorityUsageResetPeriod= +#PriorityWeightAge= +#PriorityWeightFairshare= +#PriorityWeightJobSize= +#PriorityWeightPartition= +#PriorityWeightQOS= +# +# +# LOGGING AND ACCOUNTING +#AccountingStorageEnforce=0 +#AccountingStorageHost= +#AccountingStorageLoc= +#AccountingStoragePass= +#AccountingStoragePort= +AccountingStorageType=accounting_storage/none +#AccountingStorageUser= +AccountingStoreJobComment=YES +ClusterName=cluster +#DebugFlags= +#JobCompHost= +#JobCompLoc= +#JobCompPass= +#JobCompPort= +JobCompType=jobcomp/none +#JobCompUser= +#JobContainerType=job_container/none +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/none +SlurmctldDebug=error +SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log +SlurmdDebug=error +SlurmdLogFile=/var/log/slurm-llnl/slurmd.log +#SlurmSchedLogFile= +#SlurmSchedLogLevel= +# +# +# POWER SAVE SUPPORT FOR IDLE NODES (optional) +#SuspendProgram= +#ResumeProgram= +#SuspendTimeout= +#ResumeTimeout= +#ResumeRate= +#SuspendExcNodes= +#SuspendExcParts= +#SuspendRate= +#SuspendTime= +# +# +# COMPUTE NODES +# +NodeName=nid0[0-2] REPLACE_IT State=UNKNOWN +PartitionName=all Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/examples/tutorial/dockerfiles/slurm-cluster/munge/Dockerfile b/examples/tutorial/dockerfiles/slurm-cluster/munge/Dockerfile new file mode 100644 index 0000000000..5903c261e5 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/munge/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && apt install -y \ + munge \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \ + echo "admin:admin" | chpasswd && \ + adduser admin sudo && \ + echo "admin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +RUN mkdir /scratch && \ + chown -R admin:admin /scratch + +COPY entrypoint.sh /etc/munge/ + +RUN chmod +rx /etc/munge/entrypoint.sh + +ENTRYPOINT ["/etc/munge/entrypoint.sh"] diff --git a/examples/tutorial/dockerfiles/slurm-cluster/munge/entrypoint.sh b/examples/tutorial/dockerfiles/slurm-cluster/munge/entrypoint.sh new file mode 100755 index 0000000000..c933564563 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/munge/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo /sbin/create-munge-key -f +cp /etc/munge/munge.key /scratch + +tail -f /dev/null diff --git a/examples/tutorial/dockerfiles/slurm-cluster/node/Dockerfile b/examples/tutorial/dockerfiles/slurm-cluster/node/Dockerfile new file mode 100644 index 0000000000..756a0bed86 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/node/Dockerfile @@ -0,0 +1,40 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && apt install -y \ + python3 \ + python3-pip \ + build-essential \ + libomp-dev \ + mpich \ + libmpich-dev \ + git \ + munge \ + slurm-client \ + slurmd \ + slurm-wlm-torque \ + sudo \ + wget \ + curl \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \ + echo "admin:admin" | chpasswd && \ + adduser admin sudo && \ + echo "admin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +COPY slurm.conf /etc/slurm-llnl/ +COPY cgroup.conf /etc/slurm-llnl/ +COPY docker-entrypoint.sh /etc/slurm-llnl/ + +RUN chmod +r /etc/slurm-llnl/slurm.conf +RUN chmod +r /etc/slurm-llnl/cgroup.conf +RUN chmod +rx /etc/slurm-llnl/docker-entrypoint.sh + +RUN mkdir /scratch && \ + chown -R admin:admin /scratch + +EXPOSE 6817 6818 6819 + +ENTRYPOINT ["/etc/slurm-llnl/docker-entrypoint.sh"] diff --git a/examples/tutorial/dockerfiles/slurm-cluster/node/cgroup.conf b/examples/tutorial/dockerfiles/slurm-cluster/node/cgroup.conf new file mode 100644 index 0000000000..0b634635a4 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/node/cgroup.conf @@ -0,0 +1,6 @@ +CgroupAutomount=yes +CgroupReleaseAgentDir="/etc/slurm/cgroup" +ConstrainCores=yes +ConstrainDevices=yes +ConstrainRAMSpace=yes + diff --git a/examples/tutorial/dockerfiles/slurm-cluster/node/docker-entrypoint.sh b/examples/tutorial/dockerfiles/slurm-cluster/node/docker-entrypoint.sh new file mode 100755 index 0000000000..e1a110a368 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/node/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export SLURM_CPUS_ON_NODE=$(cat /proc/cpuinfo | grep processor | wc -l) +sudo sed -i "s/REPLACE_IT/CPUs=${SLURM_CPUS_ON_NODE}/g" /etc/slurm-llnl/slurm.conf + +sudo cp /scratch/munge.key /etc/munge/munge.key +sudo service munge start +sudo slurmd -N $SLURM_NODENAME + +tail -f /dev/null diff --git a/examples/tutorial/dockerfiles/slurm-cluster/node/slurm.conf b/examples/tutorial/dockerfiles/slurm-cluster/node/slurm.conf new file mode 100644 index 0000000000..9e2c782156 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/node/slurm.conf @@ -0,0 +1,153 @@ +# slurm.conf file generated by configurator.html. +# Put this file on all nodes of your cluster. +# See the slurm.conf man page for more information. +# +SlurmctldHost=slurm-master +# +#DisableRootJobs=NO +#EnforcePartLimits=NO +#Epilog= +#EpilogSlurmctld= +#FirstJobId=1 +#MaxJobId=999999 +#GresTypes= +#GroupUpdateForce=0 +#GroupUpdateTime=600 +#JobFileAppend=0 +#JobRequeue=1 +#JobSubmitPlugins=1 +#KillOnBadExit=0 +#LaunchType=launch/slurm +#Licenses=foo*4,bar +#MailProg=/bin/mail +#MaxJobCount=5000 +#MaxStepCount=40000 +#MaxTasksPerNode=128 +MpiDefault=pmi2 +#MpiParams=ports=#-# +#PluginDir= +#PlugStackConfig= +#PrivateData=jobs +#ProctrackType=proctrack/cgroup +ProctrackType=proctrack/linuxproc +#Prolog= +#PrologFlags= +#PrologSlurmctld= +#PropagatePrioProcess=0 +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +#RebootProgram= +ReturnToService=1 +#SallocDefaultCommand= +SlurmctldPidFile=/var/run/slurmctld.pid +SlurmctldPort=6817 +SlurmdPidFile=/var/run/slurmd.pid +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=root +#SlurmdUser=root +#SrunEpilog= +#SrunProlog= +StateSaveLocation=/var/spool +SwitchType=switch/none +#TaskEpilog= +TaskPlugin=task/affinity +TaskPluginParam=Sched +#TaskProlog= +#TopologyPlugin=topology/tree +#TmpFS=/tmp +#TrackWCKey=no +#TreeWidth= +#UnkillableStepProgram= +#UsePAM=0 +# +# +# TIMERS +#BatchStartTimeout=10 +#CompleteWait=0 +#EpilogMsgTime=2000 +#GetEnvTimeout=2 +#HealthCheckInterval=0 +#HealthCheckProgram= +InactiveLimit=0 +KillWait=30 +#MessageTimeout=10 +#ResvOverRun=0 +MinJobAge=300 +#OverTimeLimit=0 +SlurmctldTimeout=120 +SlurmdTimeout=300 +#UnkillableStepTimeout=60 +#VSizeFactor=0 +Waittime=0 +# +# +# SCHEDULING +#DefMemPerCPU=0 +#MaxMemPerCPU=0 +#SchedulerTimeSlice=30 +SchedulerType=sched/backfill +SelectType=select/cons_res +SelectTypeParameters=CR_Core +# +# +# JOB PRIORITY +#PriorityFlags= +#PriorityType=priority/basic +#PriorityDecayHalfLife= +#PriorityCalcPeriod= +#PriorityFavorSmall= +#PriorityMaxAge= +#PriorityUsageResetPeriod= +#PriorityWeightAge= +#PriorityWeightFairshare= +#PriorityWeightJobSize= +#PriorityWeightPartition= +#PriorityWeightQOS= +# +# +# LOGGING AND ACCOUNTING +#AccountingStorageEnforce=0 +#AccountingStorageHost= +#AccountingStorageLoc= +#AccountingStoragePass= +#AccountingStoragePort= +AccountingStorageType=accounting_storage/none +#AccountingStorageUser= +AccountingStoreJobComment=YES +ClusterName=cluster +#DebugFlags= +#JobCompHost= +#JobCompLoc= +#JobCompPass= +#JobCompPort= +JobCompType=jobcomp/none +#JobCompUser= +#JobContainerType=job_container/none +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/none +SlurmctldDebug=error +SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log +SlurmdDebug=error +SlurmdLogFile=/var/log/slurm-llnl/slurmd.log +#SlurmSchedLogFile= +#SlurmSchedLogLevel= +# +# +# POWER SAVE SUPPORT FOR IDLE NODES (optional) +#SuspendProgram= +#ResumeProgram= +#SuspendTimeout= +#ResumeTimeout= +#ResumeRate= +#SuspendExcNodes= +#SuspendExcParts= +#SuspendRate= +#SuspendTime= +# +# +# COMPUTE NODES +# +NodeName=nid0[0-2] REPLACE_IT State=UNKNOWN +PartitionName=all Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/examples/tutorial/dockerfiles/slurm-cluster/reframe/Dockerfile b/examples/tutorial/dockerfiles/slurm-cluster/reframe/Dockerfile new file mode 100644 index 0000000000..b152b50cd4 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/reframe/Dockerfile @@ -0,0 +1,49 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && \ + apt install -y \ + build-essential \ + clang jq libomp-dev tree vim \ + git \ + mariadb-client \ + munge \ + slurm-client \ + slurm-wlm-torque \ + sudo \ + python3 \ + python3-pip \ + wget \ + curl \ + mpich \ + libmpich-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \ + echo "admin:admin" | chpasswd && adduser admin sudo && \ + echo "admin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +COPY slurm.conf /etc/slurm-llnl/ +COPY cgroup.conf /etc/slurm-llnl/ +COPY docker-entrypoint.sh /etc/slurm-llnl/ + +RUN mkdir /scratch && \ + chown -R admin:admin /scratch + +RUN chmod +rx /etc/slurm-llnl/docker-entrypoint.sh + +# Install reframe +ARG REFRAME_TAG=develop +ARG REFRAME_REPO=reframe-hpc +WORKDIR /usr/local/share +RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ + cd reframe/ && ./bootstrap.sh +ENV PATH=/usr/local/share/reframe/bin:$PATH + +WORKDIR /home/admin + +ENV USER admin +ENV SHELL bash + +ENTRYPOINT ["/etc/slurm-llnl/docker-entrypoint.sh"] diff --git a/examples/tutorial/dockerfiles/slurm-cluster/reframe/cgroup.conf b/examples/tutorial/dockerfiles/slurm-cluster/reframe/cgroup.conf new file mode 100644 index 0000000000..0b634635a4 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/reframe/cgroup.conf @@ -0,0 +1,6 @@ +CgroupAutomount=yes +CgroupReleaseAgentDir="/etc/slurm/cgroup" +ConstrainCores=yes +ConstrainDevices=yes +ConstrainRAMSpace=yes + diff --git a/examples/tutorial/dockerfiles/slurm-cluster/reframe/docker-entrypoint.sh b/examples/tutorial/dockerfiles/slurm-cluster/reframe/docker-entrypoint.sh new file mode 100755 index 0000000000..e330f5f20a --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/reframe/docker-entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +trap exit 0 INT + +while [ ! -f /scratch/munge.key ] +do + sleep 1 +done + +sudo cp /scratch/munge.key /etc/munge/munge.key +sudo service munge start +sudo sed -i "s/REPLACE_IT/CPUs=${SLURM_CPUS_ON_NODE}/g" /etc/slurm-llnl/slurm.conf + +echo "Container up and running: " +echo "==> Run 'docker exec -it /bin/bash' to run interactively" +echo "==> Press Ctrl-C to exit" +sleep infinity & +wait diff --git a/examples/tutorial/dockerfiles/slurm-cluster/reframe/slurm.conf b/examples/tutorial/dockerfiles/slurm-cluster/reframe/slurm.conf new file mode 100644 index 0000000000..9e2c782156 --- /dev/null +++ b/examples/tutorial/dockerfiles/slurm-cluster/reframe/slurm.conf @@ -0,0 +1,153 @@ +# slurm.conf file generated by configurator.html. +# Put this file on all nodes of your cluster. +# See the slurm.conf man page for more information. +# +SlurmctldHost=slurm-master +# +#DisableRootJobs=NO +#EnforcePartLimits=NO +#Epilog= +#EpilogSlurmctld= +#FirstJobId=1 +#MaxJobId=999999 +#GresTypes= +#GroupUpdateForce=0 +#GroupUpdateTime=600 +#JobFileAppend=0 +#JobRequeue=1 +#JobSubmitPlugins=1 +#KillOnBadExit=0 +#LaunchType=launch/slurm +#Licenses=foo*4,bar +#MailProg=/bin/mail +#MaxJobCount=5000 +#MaxStepCount=40000 +#MaxTasksPerNode=128 +MpiDefault=pmi2 +#MpiParams=ports=#-# +#PluginDir= +#PlugStackConfig= +#PrivateData=jobs +#ProctrackType=proctrack/cgroup +ProctrackType=proctrack/linuxproc +#Prolog= +#PrologFlags= +#PrologSlurmctld= +#PropagatePrioProcess=0 +#PropagateResourceLimits= +#PropagateResourceLimitsExcept= +#RebootProgram= +ReturnToService=1 +#SallocDefaultCommand= +SlurmctldPidFile=/var/run/slurmctld.pid +SlurmctldPort=6817 +SlurmdPidFile=/var/run/slurmd.pid +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=root +#SlurmdUser=root +#SrunEpilog= +#SrunProlog= +StateSaveLocation=/var/spool +SwitchType=switch/none +#TaskEpilog= +TaskPlugin=task/affinity +TaskPluginParam=Sched +#TaskProlog= +#TopologyPlugin=topology/tree +#TmpFS=/tmp +#TrackWCKey=no +#TreeWidth= +#UnkillableStepProgram= +#UsePAM=0 +# +# +# TIMERS +#BatchStartTimeout=10 +#CompleteWait=0 +#EpilogMsgTime=2000 +#GetEnvTimeout=2 +#HealthCheckInterval=0 +#HealthCheckProgram= +InactiveLimit=0 +KillWait=30 +#MessageTimeout=10 +#ResvOverRun=0 +MinJobAge=300 +#OverTimeLimit=0 +SlurmctldTimeout=120 +SlurmdTimeout=300 +#UnkillableStepTimeout=60 +#VSizeFactor=0 +Waittime=0 +# +# +# SCHEDULING +#DefMemPerCPU=0 +#MaxMemPerCPU=0 +#SchedulerTimeSlice=30 +SchedulerType=sched/backfill +SelectType=select/cons_res +SelectTypeParameters=CR_Core +# +# +# JOB PRIORITY +#PriorityFlags= +#PriorityType=priority/basic +#PriorityDecayHalfLife= +#PriorityCalcPeriod= +#PriorityFavorSmall= +#PriorityMaxAge= +#PriorityUsageResetPeriod= +#PriorityWeightAge= +#PriorityWeightFairshare= +#PriorityWeightJobSize= +#PriorityWeightPartition= +#PriorityWeightQOS= +# +# +# LOGGING AND ACCOUNTING +#AccountingStorageEnforce=0 +#AccountingStorageHost= +#AccountingStorageLoc= +#AccountingStoragePass= +#AccountingStoragePort= +AccountingStorageType=accounting_storage/none +#AccountingStorageUser= +AccountingStoreJobComment=YES +ClusterName=cluster +#DebugFlags= +#JobCompHost= +#JobCompLoc= +#JobCompPass= +#JobCompPort= +JobCompType=jobcomp/none +#JobCompUser= +#JobContainerType=job_container/none +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/none +SlurmctldDebug=error +SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log +SlurmdDebug=error +SlurmdLogFile=/var/log/slurm-llnl/slurmd.log +#SlurmSchedLogFile= +#SlurmSchedLogLevel= +# +# +# POWER SAVE SUPPORT FOR IDLE NODES (optional) +#SuspendProgram= +#ResumeProgram= +#SuspendTimeout= +#ResumeTimeout= +#ResumeRate= +#SuspendExcNodes= +#SuspendExcParts= +#SuspendRate= +#SuspendTime= +# +# +# COMPUTE NODES +# +NodeName=nid0[0-2] REPLACE_IT State=UNKNOWN +PartitionName=all Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/examples/tutorial/dummy/params.py b/examples/tutorial/dummy/params.py new file mode 100644 index 0000000000..aadf7f9413 --- /dev/null +++ b/examples/tutorial/dummy/params.py @@ -0,0 +1,49 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class echo_test_v0(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo' + x = parameter([0, 1]) + y = parameter([0, 1]) + + @run_after('init') + def skip_invalid(self): + self.skip_if(self.x == self.y, 'invalid parameter combination') + + @run_after('init') + def set_executable_opts(self): + self.executable_opts = [f'{self.x}', f'{self.y}'] + + @sanity_function + def validate(self): + x = sn.extractsingle(r'(\d) (\d)', self.stdout, 1, int) + y = sn.extractsingle(r'(\d) (\d)', self.stdout, 2, int) + return sn.and_(sn.assert_eq(x, self.x), sn.assert_eq(y, self.y)) + + +@rfm.simple_test +class echo_test_v1(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'echo' + xy = parameter([(0, 1), (1, 0)], fmt=lambda val: f'{val[0]}{val[1]}') + + @run_after('init') + def set_executable_opts(self): + self.x, self.y = self.xy + self.executable_opts = [f'{self.x}', f'{self.y}'] + + @sanity_function + def validate(self): + x = sn.extractsingle(r'(\d) (\d)', self.stdout, 1, int) + y = sn.extractsingle(r'(\d) (\d)', self.stdout, 2, int) + return sn.and_(sn.assert_eq(x, self.x), sn.assert_eq(y, self.y)) diff --git a/tutorials/build_systems/easybuild/eb_test.py b/examples/tutorial/easybuild/eb_test.py similarity index 100% rename from tutorials/build_systems/easybuild/eb_test.py rename to examples/tutorial/easybuild/eb_test.py diff --git a/tutorials/fixtures/osu_benchmarks.py b/examples/tutorial/mpi/osu.py similarity index 55% rename from tutorials/fixtures/osu_benchmarks.py rename to examples/tutorial/mpi/osu.py index fd31c9cb0f..2001e0d846 100644 --- a/tutorials/fixtures/osu_benchmarks.py +++ b/examples/tutorial/mpi/osu.py @@ -5,12 +5,13 @@ import os import reframe as rfm +import reframe.utility.typecheck as typ import reframe.utility.sanity as sn class fetch_osu_benchmarks(rfm.RunOnlyRegressionTest): descr = 'Fetch OSU benchmarks' - version = variable(str, value='5.6.2') + version = variable(str, value='7.3') executable = 'wget' executable_opts = [ f'http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{version}.tar.gz' # noqa: E501 @@ -41,79 +42,70 @@ def prepare_build(self): ] self.build_system.max_concurrency = 8 - @sanity_function - def validate_build(self): - # If compilation fails, the test would fail in any case, so nothing to - # further validate here. - return True - -class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): +class osu_base_test(rfm.RunOnlyRegressionTest): '''Base class of OSU benchmarks runtime tests''' - valid_systems = ['daint:gpu'] - valid_prog_environs = ['gnu', 'nvidia', 'intel'] + valid_systems = ['*'] + valid_prog_environs = ['+mpi'] num_tasks = 2 num_tasks_per_node = 1 osu_binaries = fixture(build_osu_benchmarks, scope='environment') - - @sanity_function - def validate_test(self): - return sn.assert_found(r'^8', self.stdout) - - -@rfm.simple_test -class osu_latency_test(OSUBenchmarkTestBase): - descr = 'OSU latency test' + kind = variable(str) + benchmark = variable(str) + metric = variable(typ.Str[r'latency|bandwidth']) @run_before('run') def prepare_run(self): self.executable = os.path.join( self.osu_binaries.stagedir, self.osu_binaries.build_prefix, - 'mpi', 'pt2pt', 'osu_latency' + 'c', 'mpi', self.kind, self.benchmark ) self.executable_opts = ['-x', '100', '-i', '1000'] - @performance_function('us') - def latency(self): - return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) + @sanity_function + def validate_test(self): + return sn.assert_found(r'^8', self.stdout) + def _extract_metric(self, size): + return sn.extractsingle(rf'^{size}\s+(\S+)', self.stdout, 1, float) -@rfm.simple_test -class osu_bandwidth_test(OSUBenchmarkTestBase): - descr = 'OSU bandwidth test' + @run_before('performance') + def set_perf_vars(self): + make_perf = sn.make_performance_function + if self.metric == 'latency': + self.perf_variables = { + 'latency': make_perf(self._extract_metric(8), 'us') + } + else: + self.perf_variables = { + 'bandwidth': make_perf(self._extract_metric(1048576), 'MB/s') + } - @run_before('run') - def prepare_run(self): - self.executable = os.path.join( - self.osu_binaries.stagedir, - self.osu_binaries.build_prefix, - 'mpi', 'pt2pt', 'osu_bw' - ) - self.executable_opts = ['-x', '100', '-i', '1000'] - @performance_function('MB/s') - def bandwidth(self): - return sn.extractsingle(r'^4194304\s+(\S+)', - self.stdout, 1, float) +@rfm.simple_test +class osu_latency_test(osu_base_test): + descr = 'OSU latency test' + kind = 'pt2pt/standard' + benchmark = 'osu_latency' + metric = 'latency' + executable_opts = ['-x', '3', '-i', '10'] @rfm.simple_test -class osu_allreduce_test(OSUBenchmarkTestBase): - mpi_tasks = parameter(1 << i for i in range(1, 5)) - descr = 'OSU Allreduce test' +class osu_bandwidth_test(osu_base_test): + descr = 'OSU bandwidth test' + kind = 'pt2pt/standard' + benchmark = 'osu_bw' + metric = 'bandwidth' + executable_opts = ['-x', '3', '-i', '10'] - @run_before('run') - def set_executable(self): - self.num_tasks = self.mpi_tasks - self.executable = os.path.join( - self.osu_binaries.stagedir, - self.osu_binaries.build_prefix, - 'mpi', 'collective', 'osu_allreduce' - ) - self.executable_opts = ['-m', '8', '-x', '1000', '-i', '20000'] - @performance_function('us') - def latency(self): - return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) +@rfm.simple_test +class osu_allreduce_test(osu_base_test): + descr = 'OSU Allreduce test' + kind = 'collective/blocking' + benchmark = 'osu_allreduce' + metric = 'bandwidth' + executable_opts = ['-m', '8', '-x', '3', '-i', '10'] diff --git a/examples/tutorial/mpi/osu_deps.py b/examples/tutorial/mpi/osu_deps.py new file mode 100644 index 0000000000..bed4dc6094 --- /dev/null +++ b/examples/tutorial/mpi/osu_deps.py @@ -0,0 +1,125 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os + +import reframe as rfm +import reframe.utility.sanity as sn +import reframe.utility.typecheck as typ +import reframe.utility.udeps as udeps + + +@rfm.simple_test +class fetch_osu_benchmarks(rfm.RunOnlyRegressionTest): + descr = 'Fetch OSU benchmarks' + version = variable(str, value='7.3') + executable = 'wget' + executable_opts = [ + f'http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{version}.tar.gz' # noqa: E501 + ] + local = True + valid_systems = ['pseudo-cluster:login'] + valid_prog_environs = ['gnu'] + + @sanity_function + def validate_download(self): + return sn.assert_eq(self.job.exitcode, 0) + + +@rfm.simple_test +class build_osu_benchmarks(rfm.CompileOnlyRegressionTest): + descr = 'Build OSU benchmarks' + build_system = 'Autotools' + build_prefix = variable(str) + valid_systems = ['pseudo-cluster:compute'] + valid_prog_environs = ['gnu-mpi'] + + @run_after('init') + def add_dependencies(self): + self.depends_on('fetch_osu_benchmarks', udeps.fully) + + @require_deps + def prepare_build(self, fetch_osu_benchmarks): + target = fetch_osu_benchmarks(part='login', environ='gnu') + tarball = f'osu-micro-benchmarks-{target.version}.tar.gz' + self.build_prefix = tarball[:-7] # remove .tar.gz extension + + fullpath = os.path.join(target.stagedir, tarball) + self.prebuild_cmds = [ + f'cp {fullpath} {self.stagedir}', + f'tar xzf {tarball}', + f'cd {self.build_prefix}' + ] + self.build_system.max_concurrency = 8 + + +class osu_base_test(rfm.RunOnlyRegressionTest): + '''Base class of OSU benchmarks runtime tests''' + + valid_systems = ['pseudo-cluster:compute'] + valid_prog_environs = ['gnu-mpi'] + num_tasks = 2 + num_tasks_per_node = 1 + kind = variable(str) + benchmark = variable(str) + metric = variable(typ.Str[r'latency|bandwidth']) + + @run_after('init') + def add_dependencies(self): + self.depends_on('build_osu_benchmarks', udeps.by_env) + + @require_deps + def prepare_run(self, build_osu_benchmarks): + osu_binaries = build_osu_benchmarks() + self.executable = os.path.join( + osu_binaries.stagedir, osu_binaries.build_prefix, + 'c', 'mpi', self.kind, self.benchmark + ) + self.executable_opts = ['-x', '100', '-i', '1000'] + + @sanity_function + def validate_test(self): + return sn.assert_found(r'^8', self.stdout) + + def _extract_metric(self, size): + return sn.extractsingle(rf'^{size}\s+(\S+)', self.stdout, 1, float) + + @run_before('performance') + def set_perf_vars(self): + make_perf = sn.make_performance_function + if self.metric == 'latency': + self.perf_variables = { + 'latency': make_perf(self._extract_metric(8), 'us') + } + else: + self.perf_variables = { + 'bandwidth': make_perf(self._extract_metric(1048576), 'MB/s') + } + + +@rfm.simple_test +class osu_latency_test(osu_base_test): + descr = 'OSU latency test' + kind = 'pt2pt/standard' + benchmark = 'osu_latency' + metric = 'latency' + executable_opts = ['-x', '3', '-i', '10'] + + +@rfm.simple_test +class osu_bandwidth_test(osu_base_test): + descr = 'OSU bandwidth test' + kind = 'pt2pt/standard' + benchmark = 'osu_bw' + metric = 'bandwidth' + executable_opts = ['-x', '3', '-i', '10'] + + +@rfm.simple_test +class osu_allreduce_test(osu_base_test): + descr = 'OSU Allreduce test' + kind = 'collective/blocking' + benchmark = 'osu_allreduce' + metric = 'bandwidth' + executable_opts = ['-m', '8', '-x', '3', '-i', '10'] diff --git a/tutorials/build_systems/spack/spack_test.py b/examples/tutorial/spack/spack_test.py similarity index 100% rename from tutorials/build_systems/spack/spack_test.py rename to examples/tutorial/spack/spack_test.py diff --git a/tutorials/cscs-webinar-2022/tests/src/stream.c b/examples/tutorial/stream/src/stream.c similarity index 100% rename from tutorials/cscs-webinar-2022/tests/src/stream.c rename to examples/tutorial/stream/src/stream.c diff --git a/examples/tutorial/stream/stream_build_run.py b/examples/tutorial/stream/stream_build_run.py new file mode 100644 index 0000000000..efa2174868 --- /dev/null +++ b/examples/tutorial/stream/stream_build_run.py @@ -0,0 +1,32 @@ +# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class stream_build_test(rfm.RegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/tutorials/advanced/make_test/stream_config.yaml b/examples/tutorial/stream/stream_config.yaml similarity index 89% rename from tutorials/advanced/make_test/stream_config.yaml rename to examples/tutorial/stream/stream_config.yaml index 0ad9b3c03c..66472888c2 100644 --- a/tutorials/advanced/make_test/stream_config.yaml +++ b/examples/tutorial/stream/stream_config.yaml @@ -7,11 +7,11 @@ stream_workflows: - elem_type: 'float' array_size: 16777216 num_iters: 10 - num_cpus_per_task: 4 + num_threads: 4 - elem_type: 'double' array_size: 1048576 num_iters: 100 - num_cpus_per_task: 1 + num_threads: 1 - elem_type: 'double' array_size: 16777216 num_iters: 10 diff --git a/examples/tutorial/stream/stream_cpuinfo.py b/examples/tutorial/stream/stream_cpuinfo.py new file mode 100644 index 0000000000..15b5a436d8 --- /dev/null +++ b/examples/tutorial/stream/stream_cpuinfo.py @@ -0,0 +1,51 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = variable(int, value=0) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @run_before('run') + def set_num_threads(self): + if not self.num_threads: + self.skip_if_no_procinfo() + proc = self.current_partition.processor + self.num_threads = proc.num_cores + + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_fixtures.py b/examples/tutorial/stream/stream_fixtures.py new file mode 100644 index 0000000000..86daf62340 --- /dev/null +++ b/examples/tutorial/stream/stream_fixtures.py @@ -0,0 +1,41 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_make.py b/examples/tutorial/stream/stream_make.py new file mode 100644 index 0000000000..4aedc6703f --- /dev/null +++ b/examples/tutorial/stream/stream_make.py @@ -0,0 +1,51 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'Make' + sourcesdir = 'https://github.com/jeffhammond/STREAM.git' + array_size = variable(int, value=0) + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.options = ['stream_c.exe'] + self.build_system.cflags = ['-O3', omp_flag] + if self.array_size: + self.build_system.cflags += [f'-DARRAY_SIZE={self.array_size}'] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = variable(int, value=0) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, + 'stream_c.exe') + + @run_before('run') + def set_num_threads(self): + if self.num_threads: + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_multistep.py b/examples/tutorial/stream/stream_multistep.py new file mode 100644 index 0000000000..9eb1917479 --- /dev/null +++ b/examples/tutorial/stream/stream_multistep.py @@ -0,0 +1,52 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = variable(int, value=0) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @run_before('run') + def set_num_threads(self): + if self.num_threads: + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + + @run_before('run') + def hostname_step(self): + launch_cmd = self.job.launcher.run_command(self.job) + self.prerun_cmds = [f'{launch_cmd} hostname'] + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_parameters.py b/examples/tutorial/stream/stream_parameters.py new file mode 100644 index 0000000000..0c4ac9bad0 --- /dev/null +++ b/examples/tutorial/stream/stream_parameters.py @@ -0,0 +1,51 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + array_size = variable(int, value=0) + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + if self.array_size: + self.build_system.cppflags = [f'-DARRAY_SIZE={self.array_size}'] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = parameter([1, 2, 4, 8]) + thread_placement = parameter(['close', 'cores', 'spread']) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @run_before('run') + def setup_threading(self): + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + self.env_vars['OMP_PROC_BIND'] = self.thread_placement + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_parameters_fixtures.py b/examples/tutorial/stream/stream_parameters_fixtures.py new file mode 100644 index 0000000000..c9bbbb463f --- /dev/null +++ b/examples/tutorial/stream/stream_parameters_fixtures.py @@ -0,0 +1,53 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + array_size = variable(int, value=0) + elem_type = parameter(['float', 'double']) + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + if self.array_size: + self.build_system.cppflags = [f'-DARRAY_SIZE={self.array_size}', + f'-DELEM_TYPE={self.elem_type}'] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = parameter([1, 2, 4, 8]) + thread_placement = parameter(['close', 'cores', 'spread']) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @run_before('run') + def setup_threading(self): + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + self.env_vars['OMP_PROC_BIND'] = self.thread_placement + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_runonly.py b/examples/tutorial/stream/stream_runonly.py new file mode 100644 index 0000000000..71d9ffffc7 --- /dev/null +++ b/examples/tutorial/stream/stream_runonly.py @@ -0,0 +1,25 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + executable = 'stream.x' + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_variables.py b/examples/tutorial/stream/stream_variables.py new file mode 100644 index 0000000000..a86496ffa7 --- /dev/null +++ b/examples/tutorial/stream/stream_variables.py @@ -0,0 +1,47 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = variable(int, value=0) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @run_before('run') + def set_num_threads(self): + if self.num_threads: + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/examples/tutorial/stream/stream_variables_fixtures.py b/examples/tutorial/stream/stream_variables_fixtures.py new file mode 100644 index 0000000000..c2c376e561 --- /dev/null +++ b/examples/tutorial/stream/stream_variables_fixtures.py @@ -0,0 +1,50 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class build_stream(rfm.CompileOnlyRegressionTest): + build_system = 'SingleSource' + sourcepath = 'stream.c' + executable = './stream.x' + array_size = variable(int, value=0) + + @run_before('compile') + def prepare_build(self): + omp_flag = self.current_environ.extras.get('omp_flag') + self.build_system.cflags = ['-O3', omp_flag] + if self.array_size: + self.build_system.cppflags = [f'-DARRAY_SIZE={self.array_size}'] + + +@rfm.simple_test +class stream_test(rfm.RunOnlyRegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['+openmp'] + stream_binary = fixture(build_stream, scope='environment') + num_threads = variable(int, value=0) + + @run_after('setup') + def set_executable(self): + self.executable = os.path.join(self.stream_binary.stagedir, 'stream.x') + + @run_before('run') + def set_num_threads(self): + if self.num_threads: + self.env_vars['OMP_NUM_THREADS'] = self.num_threads + + @sanity_function + def validate(self): + return sn.assert_found(r'Solution Validates', self.stdout) + + @performance_function('MB/s') + def copy_bw(self): + return sn.extractsingle(r'Copy:\s+(\S+)', self.stdout, 1, float) + + @performance_function('MB/s') + def triad_bw(self): + return sn.extractsingle(r'Triad:\s+(\S+)', self.stdout, 1, float) diff --git a/tutorials/advanced/make_test/stream_workflows.py b/examples/tutorial/stream/stream_workflows.py similarity index 70% rename from tutorials/advanced/make_test/stream_workflows.py rename to examples/tutorial/stream/stream_workflows.py index e563a51eb2..6ed33ab099 100644 --- a/tutorials/advanced/make_test/stream_workflows.py +++ b/examples/tutorial/stream/stream_workflows.py @@ -2,7 +2,6 @@ # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause - import os import yaml @@ -10,7 +9,7 @@ import reframe.core.builtins as builtins from reframe.core.meta import make_test -import stream +from stream_variables import build_stream, stream_test def load_specs(): @@ -30,24 +29,27 @@ def load_specs(): def generate_tests(specs): tests = [] for i, spec in enumerate(specs['stream_workflows']): + num_threads = spec.pop('num_threads', None) thread_scaling = spec.pop('thread_scaling', None) test_body = { - 'stream_binaries': builtins.fixture(stream.stream_build, - scope='environment', - variables=spec) + 'stream_binary': builtins.fixture(build_stream, + scope='environment', + variables=spec) } methods = [] if thread_scaling: def _set_num_threads(test): - test.num_cpus_per_task = test.num_threads + test.num_threads = test.nthr - test_body['num_threads'] = builtins.parameter(thread_scaling) + test_body['nthr'] = builtins.parameter(thread_scaling) methods.append( builtins.run_after('init')(_set_num_threads) ) + elif num_threads: + test_body['num_threads'] = num_threads tests.append(make_test( - f'stream_test_{i}', (stream.stream_test,), + f'stream_test_{i}', (stream_test,), test_body, methods )) diff --git a/reframe/__init__.py b/reframe/__init__.py index c7cb2d8139..a493c5cfd0 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -6,7 +6,7 @@ import os import sys -VERSION = '4.6.0-dev.2' +VERSION = '4.7.0-dev.2' INSTALL_PREFIX = os.path.normpath( os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) ) diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index 146560b97a..291eec50e5 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -488,7 +488,7 @@ class ConfigureBasedBuildSystem(BuildSystem): #: :default: :class:`None` builddir = variable(str, type(None), value=None) - #: Additional configuration options to be passed to the CMake invocation. + #: Additional configuration options to be passed to the configure step. #: #: :type: :class:`List[str]` #: :default: ``[]`` @@ -778,6 +778,10 @@ def post_build(self, buildjob): @property def generated_modules(self): + '''List of the EasyBuild generated modules. + + This list will be populated *after* the build succeeds. + ''' return self._eb_modules @@ -891,7 +895,6 @@ class Spack(BuildSystem): #: preinstall_cmds = variable(typ.List[str], value=[]) - def __init__(self): # Set to True if the environment was auto-generated self._auto_env = False diff --git a/reframe/core/environments.py b/reframe/core/environments.py index e5a91f9af0..e2543e0a33 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -242,6 +242,7 @@ def __init__(self, cxxflags=None, fflags=None, ldflags=None, + resources=None, **kwargs): super().__init__(name, modules, env_vars, extras, features, prepare_cmds) @@ -254,6 +255,7 @@ def __init__(self, self._cxxflags = cxxflags or [] self._fflags = fflags or [] self._ldflags = ldflags or [] + self._resources = resources or {} @property def cc(self): @@ -326,3 +328,13 @@ def nvcc(self): :type: :class:`str` ''' return self._nvcc + + @property + def resources(self): + '''The scheduler resources associated with this environment. + + .. versionadded:: 4.6.0 + + :type: :class:`Dict[str, object]` + ''' + return self._resources diff --git a/reframe/core/fixtures.py b/reframe/core/fixtures.py index c4850ae691..e285467974 100644 --- a/reframe/core/fixtures.py +++ b/reframe/core/fixtures.py @@ -407,10 +407,11 @@ class must derive from of system partition and programming environment. Since the fixture is guaranteed to have the same partition and programming environment as the parent test, the fixture class can be any derived class from - :class:`~reframe.core.pipeline.RegressionTest`. * **test**: This scope - covers a single instance of the parent test, where the resources - provided by the fixture are exclusive to each parent test instance. - The fixture class can be any derived class from + :class:`~reframe.core.pipeline.RegressionTest`. + + - **test**: This scope covers a single instance of the parent test, + where the resources provided by the fixture are exclusive to each + parent test instance. The fixture class can be any derived class from :class:`~reframe.core.pipeline.RegressionTest`. Rather than specifying the scope at the fixture class definition, ReFrame diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 0bce2bbd5b..2aa28ecd20 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -19,7 +19,7 @@ import reframe.utility.color as color import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext -from reframe.core.exceptions import ConfigError, LoggingError +from reframe.core.exceptions import ConfigError, LoggingError, what from reframe.core.warnings import suppress_deprecations from reframe.utility import is_trivially_callable from reframe.utility.profile import TimeProfiler @@ -128,14 +128,6 @@ def ignore_brokenpipe(hdlr, l): logging.Handler.handleError = handleError(logging.Handler.handleError) -def _expand_params(check): - cls = type(check) - return { - name: getattr(check, name) for name, param in cls.param_space.items - if param.is_loggable() - } - - def _guess_delim(s): '''Guess the delimiter in the given logging format string''' delims = set() @@ -169,7 +161,7 @@ def __init__(self, prefix, mode='a', encoding=None, fmt=None, # Format specifiers self.__fmt = fmt self.__perffmt = perffmt - self.__attr_patt = re.compile(r'\%\((.*?)\)s(.)?') + self.__attr_patt = re.compile(r'\%\((.*?)\)s(.*?(?=%|$))?') self.__ignore_keys = set(ignore_keys) if ignore_keys else set() def __generate_header(self, record): @@ -301,7 +293,11 @@ class CheckFieldFormatter(logging.Formatter): # NOTE: This formatter will work only for the '%' style def __init__(self, fmt=None, datefmt=None, perffmt=None, ignore_keys=None, style='%'): - super().__init__(fmt, datefmt, style) + if sys.version_info[:2] <= (3, 7): + super().__init__(fmt, datefmt, style) + else: + super().__init__(fmt, datefmt, style, + validate=(fmt != '%(check_#ALL)s')) self.__fmt = fmt self.__fmtperf = perffmt[:-1] if perffmt else '' @@ -863,6 +859,8 @@ def log_performance(self, level, task, msg=None, multiline=False): self.extra['check_partition'] = task.testcase.partition.name self.extra['check_environ'] = task.testcase.environ.name self.extra['check_result'] = 'pass' if task.succeeded else 'fail' + fail_reason = what(*task.exc_info) if not task.succeeded else None + self.extra['check_fail_reason'] = fail_reason if msg is None: msg = 'sent by ' + self.extra['osuser'] diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ef41ab7e05..462b103b4b 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -733,9 +733,7 @@ def pipeline_hooks(cls): #: .. versionadded:: 4.0.0 require_reference = variable(typ.Bool, value=False, loggable=False) - #: - #: Refer to the :doc:`ReFrame Tutorials ` for concrete usage - #: examples. + #: Patterns for checking the sanity of this test. #: #: If not set, a sanity error may be raised during sanity checking if no #: other sanity checking functions already exist. @@ -749,7 +747,7 @@ def pipeline_hooks(cls): #: The default behaviour has changed and it is now considered a #: sanity failure if this attribute is set to :class:`required`. #: - #: If a test doesn't care about its output, this must be stated + #: If a test does not care about its output, this must be stated #: explicitly as follows: #: #: :: @@ -758,6 +756,12 @@ def pipeline_hooks(cls): #: #: .. versionchanged:: 3.6 #: The default value has changed from ``None`` to ``required``. + #: + #: .. note:: + #: + #: You are advised to follow the new syntax for defining the sanity + #: check of the test using the builtin :func:`@sanity_function + #: ` decorator. sanity_patterns = variable(_DeferredExpression, loggable=False) #: Patterns for verifying the performance of this test. @@ -770,13 +774,12 @@ def pipeline_hooks(cls): #: :class:`None` is also allowed. #: :default: :class:`None` #: - #: .. warning:: + #: .. note:: #: #: You are advised to follow the new syntax for defining performance #: variables in your tests using either the :func:`@performance_function #: ` builtin or the - #: :attr:`perf_variables`, as :attr:`perf_patterns` will likely be - #: deprecated in the future. + #: :attr:`perf_variables`. perf_patterns = variable(typ.Dict[str, _DeferredExpression], type(None), loggable=False) @@ -801,9 +804,6 @@ def pipeline_hooks(cls): #: created inline using the utility #: :func:`~reframe.utility.sanity.make_performance_function`. #: - #: Refer to the :doc:`ReFrame Tutorials ` for concrete usage - #: examples. - #: #: :type: A dictionary with keys of type :class:`str` and deferred #: performance expressions as values (see #: :ref:`deferrable-performance-functions`). @@ -2018,6 +2018,7 @@ def _get_cp_env(): self._job.prepare( commands, environs, self._current_partition.prepare_cmds, + rt.runtime().get_option('general/flex_alloc_strict'), login=rt.runtime().get_option('general/0/use_login_shell'), trap_errors=rt.runtime().get_option( 'general/0/trap_job_errors' @@ -2036,7 +2037,13 @@ def _get_cp_env(): def _map_resources_to_jobopts(self): resources_opts = [] - for r, v in self.extra_resources.items(): + # Combine all resources into one dictionary, where extra_resources + # can overwrite _current_environ.resources + combined_resources = { + **self._current_environ.resources, + **self.extra_resources + } + for r, v in combined_resources.items(): resources_opts += self._current_partition.get_resource(r, **v) return resources_opts @@ -2228,14 +2235,11 @@ def check_performance(self): self.perf_variables[var] = sn.make_performance_function(expr, unit) - if self.is_dry_run(): - return - # Evaluate the performance function and retrieve the metrics with osext.change_dir(self._stagedir): for tag, expr in self.perf_variables.items(): try: - value = expr.evaluate() + value = expr.evaluate() if not self.is_dry_run() else None unit = expr.unit except Exception as e: logging.getlogger().warning( @@ -2275,27 +2279,30 @@ def check_performance(self): self._perfvalues[key] = (value, *ref, unit) - # Check the performance variables against their references. - for key, values in self._perfvalues.items(): - val, ref, low_thres, high_thres, *_ = values + if self.is_dry_run(): + return - # Verify that val is a number - if not isinstance(val, numbers.Number): - raise SanityError( - f'the value extracted for performance variable ' - f'{key!r} is not a number: {val}' - ) + # Check the performance variables against their references. + for key, values in self._perfvalues.items(): + val, ref, low_thres, high_thres, *_ = values - tag = key.split(':')[-1] - try: - sn.evaluate( - sn.assert_reference( - val, ref, low_thres, high_thres, - msg=('failed to meet reference: %s={0}, ' - 'expected {1} (l={2}, u={3})' % tag)) - ) - except SanityError as e: - raise PerformanceError(e) from None + # Verify that val is a number + if not isinstance(val, numbers.Number): + raise SanityError( + f'the value extracted for performance variable ' + f'{key!r} is not a number: {val}' + ) + + tag = key.split(':')[-1] + try: + sn.evaluate( + sn.assert_reference( + val, ref, low_thres, high_thres, + msg=('failed to meet reference: %s={0}, ' + 'expected {1} (l={2}, u={3})' % tag)) + ) + except SanityError as e: + raise PerformanceError(e) from None def _copy_job_files(self, job, dst): if job is None: @@ -2449,10 +2456,6 @@ def by_part(src, dst): def getdep(self, target, environ=None, part=None): '''Retrieve the test case of a target dependency. - This is a low-level method. The :func:`@require_deps - ` decorators should be - preferred. - :arg target: The name of the target dependency to be retrieved. :arg environ: The name of the programming environment that will be used to retrieve the test case of the target test. If ``None``, diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 0d3771d7b7..0266e43235 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -15,7 +15,7 @@ import reframe.core.shell as shell import reframe.utility.jsonext as jsonext import reframe.utility.typecheck as typ -from reframe.core.exceptions import JobError, JobNotStartedError +from reframe.core.exceptions import JobError, JobNotStartedError, SkipTestError from reframe.core.launchers import JobLauncher from reframe.core.logging import getlogger, DEBUG2 from reframe.core.meta import RegressionTestMeta @@ -550,7 +550,8 @@ def submit_time(self): ''' return self._submit_time - def prepare(self, commands, environs=None, prepare_cmds=None, **gen_opts): + def prepare(self, commands, environs=None, prepare_cmds=None, + strict_flex=False, **gen_opts): environs = environs or [] if self.num_tasks is not None and self.num_tasks <= 0: getlogger().debug(f'[F] Flexible node allocation requested') @@ -565,11 +566,12 @@ def prepare(self, commands, environs=None, prepare_cmds=None, **gen_opts): 'this scheduler backend') from e if guessed_num_tasks < min_num_tasks: - raise JobError( - 'could not satisfy the minimum task requirement: ' - 'required %s, found %s' % - (min_num_tasks, guessed_num_tasks) - ) + msg = (f'could not satisfy the minimum task requirement: ' + f'required {min_num_tasks}, found {guessed_num_tasks}') + if strict_flex: + raise JobError(msg) + else: + raise SkipTestError(msg) self.num_tasks = guessed_num_tasks getlogger().debug(f'[F] Setting num_tasks to {self.num_tasks}') diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 62d2cd2e38..47d44c1dfc 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -623,6 +623,9 @@ def poll(self, *jobs): # Join the states with ',' in case of job arrays job._state = ','.join(s.group('state') for s in job_match) + + # Use ',' to join nodes to be consistent with Slurm syntax + job._nodespec = ','.join(m.group('nodespec') for m in job_match) self._cancel_if_blocked( job, [s.group('reason') for s in state_match] ) diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 49dd38167d..3202d9c684 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -549,7 +549,8 @@ def create(cls, site_config): cflags=site_config.get(f'environments/@{e}/cflags'), cxxflags=site_config.get(f'environments/@{e}/cxxflags'), fflags=site_config.get(f'environments/@{e}/fflags'), - ldflags=site_config.get(f'environments/@{e}/ldflags') + ldflags=site_config.get(f'environments/@{e}/ldflags'), + resources=site_config.get(f'environments/@{e}/resources') ) for e in site_config.get(f'{partid}/environs') if any(re.match(pattern, e) for pattern in env_patt) ] diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f51305f4af..2fe96a3eb6 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -285,7 +285,7 @@ def main(): envvar='RFM_STAGE_DIR', configvar='systems/stagedir' ) output_options.add_argument( - '--save-log-files', action='store_true', default=False, + '--save-log-files', action='store_true', help='Save ReFrame log files to the output directory', envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) @@ -422,6 +422,12 @@ def main(): dest='flex_alloc_nodes', metavar='{all|STATE|NUM}', default=None, help='Set strategy for the flexible node allocation (default: "idle").' ) + run_options.add_argument( + '--flex-alloc-strict', action='store_true', + envvar='RFM_FLEX_ALLOC_STRICT', + configvar='general/flex_alloc_strict', + help='Fail the flexible tests if not enough nodes can be found' + ) run_options.add_argument( '-J', '--job-option', action='append', metavar='OPT', dest='job_options', default=[], @@ -474,12 +480,12 @@ def main(): # Environment options env_options.add_argument( '-M', '--map-module', action='append', metavar='MAPPING', - dest='module_mappings', default=[], + dest='module_mappings', help='Add a module mapping', envvar='RFM_MODULE_MAPPINGS ,', configvar='general/module_mappings' ) env_options.add_argument( - '-m', '--module', action='append', default=[], + '-m', '--module', action='append', metavar='MOD', dest='user_modules', help='Load module MOD before running any regression check', envvar='RFM_USER_MODULES ,', configvar='general/user_modules' @@ -501,13 +507,13 @@ def main(): envvar='RFM_NON_DEFAULT_CRAYPE', configvar='general/non_default_craype' ) env_options.add_argument( - '--purge-env', action='store_true', dest='purge_env', default=False, + '--purge-env', action='store_true', dest='purge_env', help='Unload all modules before running any regression check', envvar='RFM_PURGE_ENVIRONMENT', configvar='general/purge_environment' ) env_options.add_argument( '-u', '--unload-module', action='append', metavar='MOD', - dest='unload_modules', default=[], + dest='unload_modules', help='Unload module MOD before running any regression check', envvar='RFM_UNLOAD_MODULES ,', configvar='general/unload_modules' ) @@ -1433,7 +1439,7 @@ def module_unuse(*paths): printer, options.duration or options.reruns ) - if options.performance_report: + if options.performance_report and not options.dry_run: printer.info(runner.stats.performance_report()) # Generate the report for this session diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index f28cf917fa..dd95746987 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import abc +import contextlib import copy import os import signal @@ -328,6 +329,27 @@ def __exit__(this, exc_type, exc_value, traceback): self.fail() raise TaskExit from e + def _dry_run_call(self, fn, *args, **kwargs): + '''Call check's fn method in dry-run mode.''' + + @contextlib.contextmanager + def temp_dry_run(check): + dry_run_save = check._rfm_dry_run + try: + check._rfm_dry_run = True + yield check + except ABORT_REASONS: + raise + except BaseException: + pass + finally: + check._rfm_dry_run = dry_run_save + + with runtime.temp_config(self.testcase.partition.fullname): + with temp_dry_run(self.check): + return fn(*args, **kwargs) + + @logging.time_function def setup(self, *args, **kwargs): self.testcase.prepare() @@ -372,12 +394,15 @@ def run_wait(self): @logging.time_function def sanity(self): + self._perflogger = logging.getperflogger(self.check) self._safe_call(self.check.sanity) @logging.time_function - def performance(self): - self._perflogger = logging.getperflogger(self.check) - self._safe_call(self.check.performance) + def performance(self, dry_run=False): + if dry_run: + self._dry_run_call(self.check.performance) + else: + self._safe_call(self.check.performance) @logging.time_function def finalize(self): diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index e6dc203f92..b75588c5c3 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -199,6 +199,10 @@ def on_task_failure(self, task): self.printer.status('FAIL', msg, just='right') _print_perf(task) + if task.failed_stage == 'sanity': + # Dry-run the performance stage to trigger performance logging + task.performance(dry_run=True) + timings = task.pipeline_timings(['setup', 'compile_complete', 'run_complete', @@ -617,6 +621,10 @@ def on_task_failure(self, task): self.printer.status('FAIL', msg, just='right') _print_perf(task) + if task.failed_stage == 'sanity': + # Dry-run the performance stage to trigger performance logging + task.performance(dry_run=True) + timings = task.pipeline_timings(['setup', 'compile_complete', 'run_complete', diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 638cfaf7be..61d9c58e18 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -411,6 +411,16 @@ "type": "array", "items": {"$ref": "#/defs/alphanum_string"} }, + "resources": { + "type": "object", + "propertyNames": { + "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" + }, + "additionalProperties": { + "type": "object", + "additionalProperties": true + } + }, "target_systems": {"$ref": "#/defs/system_ref"} }, "required": ["name"], diff --git a/reframe/utility/cpuinfo.py b/reframe/utility/cpuinfo.py index 4f1e149d5e..1e7ea45b4a 100644 --- a/reframe/utility/cpuinfo.py +++ b/reframe/utility/cpuinfo.py @@ -284,8 +284,7 @@ def cpuinfo(): ret = { 'arch': archspec.cpu.host().name, 'vendor': archspec.cpu.host().vendor, - 'model': archspec.cpu.detect.raw_info_dictionary().get('model name', - 'N/A'), + 'model': archspec.cpu.brand_string() or 'N/A', 'platform': platform.machine() } diff --git a/reframe/utility/osext.py b/reframe/utility/osext.py index 2f45fa0e9c..75f057220a 100644 --- a/reframe/utility/osext.py +++ b/reframe/utility/osext.py @@ -692,7 +692,7 @@ def git_repo_exists(url, timeout=5): return True -def git_repo_hash(commit='HEAD', short=True, wd=None): +def git_repo_hash(commit='HEAD', short=True, wd='.'): '''Return the SHA1 hash of a Git commit. :arg commit: The commit to look at. @@ -700,19 +700,18 @@ def git_repo_hash(commit='HEAD', short=True, wd=None): characters of the long hash. We don't rely on Git for the short hash, since depending on the version it might return either 7 or 8 characters. - :arg wd: Change to this directory before retrieving the hash. If ``None``, - ReFrame's install prefix will be used. + :arg wd: Change to this directory before retrieving the hash. :returns: The Git commit hash or ``None`` if the hash could not be retrieved. + + .. versionchanged:: 4.6.1 + Default working directory is now ``.``. ''' try: - wd = wd or reframe.INSTALL_PREFIX - with change_dir(wd): - # Do not log this command, since we need to call this function - # from the logger - completed = run_command(f'git rev-parse {commit}', - check=True, log=False) - + # Do not log this command, since we need to call this function + # from the logger + completed = run_command(f'git -C {wd} rev-parse {commit}', + check=True, log=False) except (SpawnedProcessError, FileNotFoundError): return None @@ -730,13 +729,14 @@ def reframe_version(): If ReFrame's installation contains the repository metadata and the current version is a pre-release version, the repository's hash will be appended to the actual version. - ''' - repo_hash = git_repo_hash() - if repo_hash and semver.VersionInfo.parse(reframe.VERSION).prerelease: - return f'{reframe.VERSION}+{repo_hash}' - else: - return reframe.VERSION + if (semver.VersionInfo.parse(reframe.VERSION).prerelease and + os.path.exists(os.path.join(reframe.INSTALL_PREFIX, '.git'))): + repo_hash = git_repo_hash(wd=reframe.INSTALL_PREFIX) + if repo_hash: + return f'{reframe.VERSION}+{repo_hash}' + + return reframe.VERSION def expandvars(s): diff --git a/requirements.txt b/requirements.txt index c5ecc01541..72d0ecac6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ -archspec==0.2.2 +archspec==0.2.4 argcomplete==3.1.2; python_version < '3.8' -argcomplete==3.2.3; python_version >= '3.8' +argcomplete==3.3.0; python_version >= '3.8' importlib_metadata==4.0.1; python_version < '3.8' jsonschema==3.2.0 -lxml==5.2.0 +lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64' +lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64' pytest==7.0.1; python_version < '3.8' -pytest==8.1.1; python_version >= '3.8' +pytest==8.2.1; python_version >= '3.8' pytest-forked==1.4.0; python_version == '3.6' pytest-forked==1.6.0; python_version >= '3.7' pytest-parallel==0.1.1 @@ -14,11 +15,12 @@ pytest-rerunfailures==13.0; python_version == '3.7' pytest-rerunfailures==14.0; python_version >= '3.8' PyYAML==6.0.1 requests==2.27.1; python_version == '3.6' -requests==2.31.0; python_version >= '3.7' +requests==2.31.0; python_version == '3.7' +requests==2.32.2; python_version >= '3.8' semver==2.13.0; python_version == '3.6' semver==3.0.2; python_version >= '3.7' setuptools==59.6.0; python_version == '3.6' setuptools==68.0.0; python_version == '3.7' -setuptools==69.2.0; python_version >= '3.8' +setuptools==70.0.0; python_version >= '3.8' wcwidth==0.2.13 #+pygelf%pygelf==0.4.0 diff --git a/setup.cfg b/setup.cfg index e4fd492e25..cbbf1b17e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,11 +26,12 @@ packages = find_namespace: python_requires = >=3.6 scripts = bin/reframe install_requires = - archspec <= 0.2.2 + archspec >= 0.2.4 argcomplete argcomplete <= 3.1.2; python_version < '3.8' jsonschema - lxml + lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64' + lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64' PyYAML requests requests <= 2.27.1; python_version == '3.6' diff --git a/tutorials/advanced/affinity/affinity.py b/tutorials/advanced/affinity/affinity.py deleted file mode 100644 index 363fbd9047..0000000000 --- a/tutorials/advanced/affinity/affinity.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class AffinityTest(rfm.RegressionTest): - valid_systems = ['daint:gpu', 'daint:mc'] - valid_prog_environs = ['*'] - sourcesdir = 'https://github.com/vkarak/affinity.git' - build_system = 'Make' - executable = './affinity' - - @run_before('compile') - def set_build_system_options(self): - self.build_system.options = ['OPENMP=1'] - - @run_before('run') - def set_cpu_binding(self): - self.job.launcher.options = ['--cpu-bind=cores'] - - @sanity_function - def validate_test(self): - return sn.assert_found(r'CPU affinity', self.stdout) diff --git a/tutorials/advanced/containers/gromacs_test.py b/tutorials/advanced/containers/gromacs_test.py deleted file mode 100644 index 7c46eb4103..0000000000 --- a/tutorials/advanced/containers/gromacs_test.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -from hpctestlib.sciapps.gromacs.benchmarks import gromacs_check - - -def _hecbiosim_bench(params): - for p in params: - if p[0] == 'HECBioSim/hEGFRDimerSmallerPL': - return [p] - - -@rfm.simple_test -class gromacs_containerized_test(gromacs_check): - # Restrict library test parameters to only those relevant for this example - benchmark_info = parameter(inherit_params=True, - filter_params=_hecbiosim_bench, - fmt=lambda x: x[0]) - nb_impl = parameter(['gpu']) - - # New parameter for testing the various images - gromacs_image = parameter([ - None, - 'nvcr.io/hpc/gromacs:2020', - 'nvcr.io/hpc/gromacs:2020.2', - 'nvcr.io/hpc/gromacs:2021', - 'nvcr.io/hpc/gromacs:2021.3', - 'nvcr.io/hpc/gromacs:2022.1' - ]) - valid_systems = ['daint:gpu'] - valid_prog_environs = ['gnu'] - use_multithreading = False - executable = 'gmx mdrun' - executable_opts += ['-dlb yes', '-ntomp 12', '-npme -1', '-v'] - num_tasks = 1 - num_tasks_per_node = 1 - num_cpus_per_task = 12 - - @run_after('init') - def setup_container_run(self): - exec_cmd = ' '.join([self.executable, *self.executable_opts]) - self.container_platform.image = self.gromacs_image - self.container_platform.command = exec_cmd - if self.gromacs_image is None: - self.modules = ['daint-gpu', 'GROMACS'] diff --git a/tutorials/advanced/flexnodes/flextest.py b/tutorials/advanced/flexnodes/flextest.py deleted file mode 100644 index 415d7b9109..0000000000 --- a/tutorials/advanced/flexnodes/flextest.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class HostnameCheck(rfm.RunOnlyRegressionTest): - valid_systems = ['daint:gpu', 'daint:mc'] - valid_prog_environs = ['cray'] - executable = 'hostname' - num_tasks = 0 - num_tasks_per_node = 1 - - @sanity_function - def validate_test(self): - return sn.assert_eq( - self.num_tasks, - sn.count(sn.findall(r'^nid\d+$', self.stdout)) - ) diff --git a/tutorials/advanced/jobopts/eatmemory.py b/tutorials/advanced/jobopts/eatmemory.py deleted file mode 100644 index 5b9828c0b1..0000000000 --- a/tutorials/advanced/jobopts/eatmemory.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class MemoryLimitTest(rfm.RegressionTest): - valid_systems = ['daint:gpu', 'daint:mc'] - valid_prog_environs = ['gnu'] - sourcepath = 'eatmemory.c' - executable_opts = ['2000M'] - - @run_before('run') - def set_memory_limit(self): - self.job.options = ['--mem=1000'] - - @sanity_function - def validate_test(self): - return sn.assert_found( - r'(exceeded memory limit)|(Out Of Memory)', self.stderr - ) - - -@rfm.simple_test -class MemoryLimitWithResourcesTest(rfm.RegressionTest): - valid_systems = ['daint:gpu', 'daint:mc'] - valid_prog_environs = ['gnu'] - sourcepath = 'eatmemory.c' - executable_opts = ['2000M'] - extra_resources = { - 'memory': {'size': '1000'} - } - - @sanity_function - def validate_test(self): - return sn.assert_found( - r'(exceeded memory limit)|(Out Of Memory)', self.stderr - ) diff --git a/tutorials/advanced/jobopts/src/eatmemory.c b/tutorials/advanced/jobopts/src/eatmemory.c deleted file mode 100644 index 64db768382..0000000000 --- a/tutorials/advanced/jobopts/src/eatmemory.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * File: eatmemory.c - * Author: Julio Viera - * - * Created on August 27, 2012, 2:23 PM - */ - -#include -#include -#include -#include -#include -#include - -#if defined(_SC_PHYS_PAGES) && defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGE_SIZE) -#define MEMORY_PERCENTAGE -#endif - -#ifdef MEMORY_PERCENTAGE -size_t getTotalSystemMemory(){ - long pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGE_SIZE); - return pages * page_size; -} - -size_t getFreeSystemMemory(){ - long pages = sysconf(_SC_AVPHYS_PAGES); - long page_size = sysconf(_SC_PAGE_SIZE); - return pages * page_size; -} -#endif - -bool eat(long total,int chunk){ - long i; - for(i=0;i\n"); - printf("Size can be specified in megabytes or gigabytes in the following way:\n"); - printf("# # Bytes example: 1024\n"); - printf("#M # Megabytes example: 15M\n"); - printf("#G # Gigabytes example: 2G\n"); -#ifdef MEMORY_PERCENTAGE - printf("#%% # Percent example: 50%%\n"); -#endif - printf("\n"); - }else if(i>0){ - int len=strlen(arg); - char unit=arg[len - 1]; - long size=-1; - int chunk=1024; - if(!isdigit(unit) ){ - if(unit=='M' || unit=='G'){ - arg[len-1]=0; - size=atol(arg) * (unit=='M'?1024*1024:1024*1024*1024); - } -#ifdef MEMORY_PERCENTAGE - else if (unit=='%') { - size = (atol(arg) * (long)getFreeSystemMemory())/100; - } -#endif - else{ - printf("Invalid size format\n"); - exit(0); - } - }else{ - size=atoi(arg); - } - printf("Eating %ld bytes in chunks of %d...\n",size,chunk); - if(eat(size,chunk)){ - if(isatty(fileno(stdin))) { - printf("Done, press any key to free the memory\n"); - getchar(); - } else { - printf("Done, kill this process to free the memory\n"); - while(true) { - sleep(1); - } - } - }else{ - printf("ERROR: Could not allocate the memory"); - } - } - } - -} diff --git a/tutorials/advanced/library/lib/__init__.py b/tutorials/advanced/library/lib/__init__.py deleted file mode 100644 index 78395f3ac3..0000000000 --- a/tutorials/advanced/library/lib/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -class ContainerBase(rfm.RunOnlyRegressionTest, pin_prefix=True): - '''Test that asserts the ubuntu version of the image.''' - - # Derived tests must override this parameter - platform = parameter() - image_prefix = variable(str, value='') - - # Parametrize the test on two different versions of ubuntu. - dist = parameter(['18.04', '20.04']) - dist_name = variable(dict, value={ - '18.04': 'Bionic Beaver', - '20.04': 'Focal Fossa', - }) - - @run_after('setup') - def set_description(self): - self.descr = ( - f'Run commands inside a container using ubuntu {self.dist}' - ) - - @run_before('run') - def set_container_platform(self): - self.container_platform = self.platform - self.container_platform.image = ( - f'{self.image_prefix}ubuntu:{self.dist}' - ) - self.container_platform.command = ( - "bash -c /rfm_workdir/get_os_release.sh" - ) - - @property - def os_release_pattern(self): - name = self.dist_name[self.dist] - return rf'{self.dist}.\d+ LTS \({name}\)' - - @sanity_function - def assert_release(self): - return sn.all([ - sn.assert_found(self.os_release_pattern, 'release.txt'), - sn.assert_found(self.os_release_pattern, self.stdout) - ]) diff --git a/tutorials/advanced/library/lib/src/get_os_release.sh b/tutorials/advanced/library/lib/src/get_os_release.sh deleted file mode 100755 index 9ca7b5c009..0000000000 --- a/tutorials/advanced/library/lib/src/get_os_release.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cat /etc/os-release | tee /rfm_workdir/release.txt diff --git a/tutorials/advanced/library/usr/container_test.py b/tutorials/advanced/library/usr/container_test.py deleted file mode 100644 index c8852a5700..0000000000 --- a/tutorials/advanced/library/usr/container_test.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import tutorials.advanced.library.lib as lib - - -@rfm.simple_test -class ContainerTest(lib.ContainerBase): - platform = parameter(['Sarus', 'Singularity']) - valid_systems = ['daint:gpu'] - valid_prog_environs = ['builtin'] - - @run_after('setup') - def set_image_prefix(self): - if self.platform == 'Singularity': - self.image_prefix = 'docker://' diff --git a/tutorials/advanced/make_test/stream.py b/tutorials/advanced/make_test/stream.py deleted file mode 100644 index 23a3ad08ed..0000000000 --- a/tutorials/advanced/make_test/stream.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Claus - -import os -import reframe as rfm -import reframe.utility.sanity as sn - - -class stream_build(rfm.CompileOnlyRegressionTest): - build_system = 'SingleSource' - sourcepath = 'stream.c' - array_size = variable(int, value=(1 << 25)) - num_iters = variable(int, value=10) - elem_type = variable(str, value='double') - executable = 'stream' - - @run_before('compile') - def setup_build(self): - try: - omp_flag = self.current_environ.extras['ompflag'] - except KeyError: - envname = self.current_environ.name - self.skip(f'"ompflag" not defined for enviornment {envname!r}') - - self.build_system.cflags = [omp_flag, '-O3'] - self.build_system.cppflags = [f'-DSTREAM_ARRAY_SIZE={self.array_size}', - f'-DNTIMES={self.num_iters}', - f'-DSTREAM_TYPE={self.elem_type}'] - - @sanity_function - def validate_build(self): - return True - - -@rfm.simple_test -class stream_test(rfm.RunOnlyRegressionTest): - stream_binaries = fixture(stream_build, scope='environment') - valid_systems = ['*'] - valid_prog_environs = ['+openmp'] - - @run_before('run') - def setup_omp_env(self): - self.executable = os.path.join(self.stream_binaries.stagedir, 'stream') - procinfo = self.current_partition.processor - self.num_cpus_per_task = procinfo.num_cores - self.env_vars = { - 'OMP_NUM_THREADS': self.num_cpus_per_task, - 'OMP_PLACES': 'cores' - } - - @sanity_function - def validate_solution(self): - return sn.assert_found(r'Solution Validates', self.stdout) - - @performance_function('MB/s') - def copy_bandwidth(self): - return sn.extractsingle(r'Copy:\s+(\S+)\s+.*', self.stdout, 1, float) diff --git a/tutorials/advanced/makefiles/maketest.py b/tutorials/advanced/makefiles/maketest.py deleted file mode 100644 index 84d9e3ff3d..0000000000 --- a/tutorials/advanced/makefiles/maketest.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class MakefileTest(rfm.RegressionTest): - elem_type = parameter(['float', 'double']) - - descr = 'Test demonstrating use of Makefiles' - valid_systems = ['*'] - valid_prog_environs = ['clang', 'gnu'] - executable = './dotprod' - executable_opts = ['100000'] - build_system = 'Make' - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = [f'-DELEM_TYPE={self.elem_type}'] - - @sanity_function - def validate_test(self): - return sn.assert_found(rf'Result \({self.elem_type}\):', self.stdout) - - -@rfm.simple_test -class MakeOnlyTest(rfm.CompileOnlyRegressionTest): - elem_type = parameter(['float', 'double']) - descr = 'Test demonstrating use of Makefiles' - valid_systems = ['*'] - valid_prog_environs = ['clang', 'gnu'] - build_system = 'Make' - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = [f'-DELEM_TYPE={self.elem_type}'] - - @sanity_function - def validate_compilation(self): - return sn.assert_not_found(r'warning', self.stdout) diff --git a/tutorials/advanced/makefiles/maketest_mixin.py b/tutorials/advanced/makefiles/maketest_mixin.py deleted file mode 100644 index a813e44725..0000000000 --- a/tutorials/advanced/makefiles/maketest_mixin.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - - -import reframe as rfm -import reframe.utility.sanity as sn - - -class ElemTypeParam(rfm.RegressionMixin): - elem_type = parameter(['float', 'double']) - - -@rfm.simple_test -class MakefileTestAlt(rfm.RegressionTest, ElemTypeParam): - descr = 'Test demonstrating use of Makefiles' - valid_systems = ['*'] - valid_prog_environs = ['clang', 'gnu'] - executable = './dotprod' - executable_opts = ['100000'] - build_system = 'Make' - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = [f'-DELEM_TYPE={self.elem_type}'] - - @sanity_function - def validate_test(self): - return sn.assert_found( - rf'Result \({self.elem_type}\):', self.stdout - ) - - -@rfm.simple_test -class MakeOnlyTestAlt(rfm.CompileOnlyRegressionTest, ElemTypeParam): - descr = 'Test demonstrating use of Makefiles' - valid_systems = ['*'] - valid_prog_environs = ['clang', 'gnu'] - build_system = 'Make' - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = [f'-DELEM_TYPE={self.elem_type}'] - - @sanity_function - def validate_build(self): - return sn.assert_not_found(r'warning', self.stdout) diff --git a/tutorials/advanced/makefiles/src/Makefile b/tutorials/advanced/makefiles/src/Makefile deleted file mode 100644 index d156e901d7..0000000000 --- a/tutorials/advanced/makefiles/src/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -EXECUTABLE := dotprod - -.SUFFIXES: .o .cpp - -CXXFLAGS ?= -std=c++11 -OBJS := dotprod.o -$(EXECUTABLE): $(OBJS) - $(CXX) $(LDFLAGS) -o $@ $^ - -$(OBJS): dotprod.cpp - $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $(LDFLAGS) -o $@ $^ diff --git a/tutorials/advanced/makefiles/src/dotprod.cpp b/tutorials/advanced/makefiles/src/dotprod.cpp deleted file mode 100644 index ae8bf2f56c..0000000000 --- a/tutorials/advanced/makefiles/src/dotprod.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -// ReFrame Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include - -#ifndef ELEM_TYPE -#define ELEM_TYPE double -#endif - -using elem_t = ELEM_TYPE; - -template -T dotprod(const std::vector &x, const std::vector &y) -{ - assert(x.size() == y.size()); - T sum = 0; - for (std::size_t i = 0; i < x.size(); ++i) { - sum += x[i] * y[i]; - } - - return sum; -} - -template -struct type_name { - static constexpr const char *value = nullptr; -}; - -template<> -struct type_name { - static constexpr const char *value = "float"; -}; - -template<> -struct type_name { - static constexpr const char *value = "double"; -}; - -int main(int argc, char *argv[]) -{ - if (argc < 2) { - std::cerr << argv[0] << ": too few arguments\n"; - std::cerr << "Usage: " << argv[0] << " DIM\n"; - return 1; - } - - std::size_t N = std::atoi(argv[1]); - if (N < 0) { - std::cerr << argv[0] - << ": array dimension must a positive integer: " << argv[1] - << "\n"; - return 1; - } - - std::vector x(N), y(N); - std::random_device seed; - std::mt19937 rand(seed()); - std::uniform_real_distribution<> dist(-1, 1); - for (std::size_t i = 0; i < N; ++i) { - x[i] = dist(rand); - y[i] = dist(rand); - } - - std::cout << "Result (" << type_name::value << "): " - << dotprod(x, y) << "\n"; - return 0; -} diff --git a/tutorials/advanced/multilaunch/multilaunch.py b/tutorials/advanced/multilaunch/multilaunch.py deleted file mode 100644 index af62dac2e0..0000000000 --- a/tutorials/advanced/multilaunch/multilaunch.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class MultiLaunchTest(rfm.RunOnlyRegressionTest): - valid_systems = ['daint:gpu', 'daint:mc'] - valid_prog_environs = ['builtin'] - executable = 'hostname' - num_tasks = 4 - num_tasks_per_node = 1 - - @run_before('run') - def pre_launch(self): - cmd = self.job.launcher.run_command(self.job) - self.prerun_cmds = [ - f'{cmd} -n {n} {self.executable}' - for n in range(1, self.num_tasks) - ] - - @sanity_function - def validate_test(self): - return sn.assert_eq( - sn.count(sn.extractall(r'^nid\d+', self.stdout)), 10 - ) diff --git a/tutorials/advanced/parameterized/stream.py b/tutorials/advanced/parameterized/stream.py deleted file mode 100644 index 5754a85412..0000000000 --- a/tutorials/advanced/parameterized/stream.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class StreamMultiSysTest(rfm.RegressionTest): - num_bytes = parameter(1 << pow for pow in range(19, 30)) - array_size = variable(int) - ntimes = variable(int) - - valid_systems = ['*'] - valid_prog_environs = ['cray', 'gnu', 'intel', 'nvidia'] - prebuild_cmds = [ - 'wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c' # noqa: E501 - ] - build_system = 'SingleSource' - sourcepath = 'stream.c' - env_vars = { - 'OMP_NUM_THREADS': '4', - 'OMP_PLACES': 'cores' - } - reference = { - '*': { - 'Triad': (0, None, None, 'MB/s'), - } - } - - # Flags per programming environment - flags = variable(dict, value={ - 'cray': ['-fopenmp', '-O3', '-Wall'], - 'gnu': ['-fopenmp', '-O3', '-Wall'], - 'intel': ['-qopenmp', '-O3', '-Wall'], - 'nvidia': ['-mp', '-O3'] - }) - - # Number of cores for each system - cores = variable(dict, value={ - 'catalina:default': 4, - 'daint:gpu': 12, - 'daint:mc': 36, - 'daint:login': 10 - }) - - @run_after('init') - def setup_build(self): - self.array_size = (self.num_bytes >> 3) // 3 - self.ntimes = 100*1024*1024 // self.array_size - self.descr = ( - f'STREAM test (array size: {self.array_size}, ' - f'ntimes: {self.ntimes})' - ) - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = [f'-DSTREAM_ARRAY_SIZE={self.array_size}', - f'-DNTIMES={self.ntimes}'] - environ = self.current_environ.name - self.build_system.cflags = self.flags.get(environ, []) - - @run_before('run') - def set_num_threads(self): - num_threads = self.cores.get(self.current_partition.fullname, 1) - self.num_cpus_per_task = num_threads - self.env_vars = { - 'OMP_NUM_THREADS': num_threads, - 'OMP_PLACES': 'cores' - } - - @sanity_function - def validate_solution(self): - return sn.assert_found(r'Solution Validates', self.stdout) - - @performance_function('MB/s', perf_key='Triad') - def extract_triad_bw(self): - return sn.extractsingle(r'Triad:\s+(\S+)\s+.*', self.stdout, 1, float) diff --git a/tutorials/advanced/random/prepostrun.py b/tutorials/advanced/random/prepostrun.py deleted file mode 100644 index 054d504390..0000000000 --- a/tutorials/advanced/random/prepostrun.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class PrepostRunTest(rfm.RunOnlyRegressionTest): - descr = 'Pre- and post-run demo test' - valid_systems = ['*'] - valid_prog_environs = ['*'] - prerun_cmds = ['source limits.sh'] - postrun_cmds = ['echo FINISHED'] - executable = './random_numbers.sh' - - @sanity_function - def validate_test(self): - numbers = sn.extractall( - r'Random: (?P\S+)', self.stdout, 'number', float - ) - return sn.all([ - sn.assert_eq(sn.count(numbers), 100), - sn.all(sn.map(lambda x: sn.assert_bounded(x, 90, 100), numbers)), - sn.assert_found(r'FINISHED', self.stdout) - ]) diff --git a/tutorials/advanced/random/randint.py b/tutorials/advanced/random/randint.py deleted file mode 100644 index 1982d3ef64..0000000000 --- a/tutorials/advanced/random/randint.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class DeferredIterationTest(rfm.RunOnlyRegressionTest): - descr = 'Apply a sanity function iteratively' - valid_systems = ['*'] - valid_prog_environs = ['*'] - executable = './random_numbers.sh' - - @sanity_function - def validate_test(self): - numbers = sn.extractall( - r'Random: (?P\S+)', self.stdout, 'number', float - ) - return sn.all([ - sn.assert_eq(sn.count(numbers), 100), - sn.all(sn.map(lambda x: sn.assert_bounded(x, 90, 100), numbers)) - ]) diff --git a/tutorials/advanced/random/src/limits.sh b/tutorials/advanced/random/src/limits.sh deleted file mode 100644 index e440f79269..0000000000 --- a/tutorials/advanced/random/src/limits.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -export LOWER=90 -export UPPER=100 diff --git a/tutorials/advanced/random/src/random_numbers.sh b/tutorials/advanced/random/src/random_numbers.sh deleted file mode 100755 index 35f07d1837..0000000000 --- a/tutorials/advanced/random/src/random_numbers.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# rfmdocstart: random_numbers -if [ -z $LOWER ]; then - export LOWER=90 -fi - -if [ -z $UPPER ]; then - export UPPER=100 -fi - -for i in {1..100}; do - echo Random: $((RANDOM%($UPPER+1-$LOWER)+$LOWER)) -done -# rfmdocend: random_numbers diff --git a/tutorials/advanced/runonly/echorand.py b/tutorials/advanced/runonly/echorand.py deleted file mode 100644 index 49d4287c52..0000000000 --- a/tutorials/advanced/runonly/echorand.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class EchoRandTest(rfm.RunOnlyRegressionTest): - descr = 'A simple test that echoes a random number' - valid_systems = ['*'] - valid_prog_environs = ['*'] - lower = variable(int, value=90) - upper = variable(int, value=100) - executable = 'echo' - executable_opts = [ - 'Random: ', - f'$((RANDOM%({upper}+1-{lower})+{lower}))' - ] - - @sanity_function - def assert_solution(self): - return sn.assert_bounded( - sn.extractsingle( - r'Random: (?P\S+)', self.stdout, 'number', float - ), - self.lower, self.upper - ) diff --git a/tutorials/advanced/user_imports/commonutil/__init__.py b/tutorials/advanced/user_imports/commonutil/__init__.py deleted file mode 100644 index 83efa343c9..0000000000 --- a/tutorials/advanced/user_imports/commonutil/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -def greetings(name): - return f'Hello, {name}' diff --git a/tutorials/advanced/user_imports/tests/test.py b/tutorials/advanced/user_imports/tests/test.py deleted file mode 100644 index 5eb7ce0ffa..0000000000 --- a/tutorials/advanced/user_imports/tests/test.py +++ /dev/null @@ -1,28 +0,0 @@ -import reframe as rfm -import reframe.utility as util -import reframe.utility.sanity as sn -from testutil import greetings_from_test - -commonutil = util.import_module('..commonutil') - - -@rfm.simple_test -class MyTest(rfm.RunOnlyRegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - executable = f'echo {commonutil.greetings("friends")}' - - @sanity_function - def validate(self): - return sn.assert_found('Hello, friends', self.stdout) - - -@rfm.simple_test -class MyTest2(MyTest): - @run_before('run') - def set_exec(self): - self.executable = f'echo {greetings_from_test(self)}' - - @sanity_function - def validate(self): - return sn.assert_found(f'Hello from {self.name}', self.stdout) diff --git a/tutorials/advanced/user_imports/tests/testutil.py b/tutorials/advanced/user_imports/tests/testutil.py deleted file mode 100644 index a53bba8961..0000000000 --- a/tutorials/advanced/user_imports/tests/testutil.py +++ /dev/null @@ -1,2 +0,0 @@ -def greetings_from_test(test): - return f'Hello from {test.name}' diff --git a/tutorials/basics/hello/hello1.py b/tutorials/basics/hello/hello1.py deleted file mode 100644 index 9f384ae546..0000000000 --- a/tutorials/basics/hello/hello1.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class HelloTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - sourcepath = 'hello.c' - - @sanity_function - def assert_hello(self): - return sn.assert_found(r'Hello, World\!', self.stdout) diff --git a/tutorials/basics/hello/hello2.py b/tutorials/basics/hello/hello2.py deleted file mode 100644 index 6e4d4a10c1..0000000000 --- a/tutorials/basics/hello/hello2.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class HelloMultiLangTest(rfm.RegressionTest): - lang = parameter(['c', 'cpp']) - - valid_systems = ['*'] - valid_prog_environs = ['*'] - - @run_before('compile') - def set_sourcepath(self): - self.sourcepath = f'hello.{self.lang}' - - @sanity_function - def assert_hello(self): - return sn.assert_found(r'Hello, World\!', self.stdout) diff --git a/tutorials/basics/hello/src/hello.c b/tutorials/basics/hello/src/hello.c deleted file mode 100644 index 6910f5f791..0000000000 --- a/tutorials/basics/hello/src/hello.c +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -// ReFrame Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: BSD-3-Clause - -#include - - -int main() -{ - printf("Hello, World!\n"); - return 0; -} diff --git a/tutorials/basics/hello/src/hello.cpp b/tutorials/basics/hello/src/hello.cpp deleted file mode 100644 index 62e08f9d51..0000000000 --- a/tutorials/basics/hello/src/hello.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -// ReFrame Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: BSD-3-Clause - -#include - - -int main() -{ - std::cout << "Hello, World!\n"; - return 0; -} diff --git a/tutorials/basics/hellomp/hellomp1.py b/tutorials/basics/hellomp/hellomp1.py deleted file mode 100644 index c00f85f9ae..0000000000 --- a/tutorials/basics/hellomp/hellomp1.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class HelloThreadedTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - sourcepath = 'hello_threads.cpp' - build_system = 'SingleSource' - executable_opts = ['16'] - - @run_before('compile') - def set_compilation_flags(self): - self.build_system.cxxflags = ['-std=c++11', '-Wall'] - environ = self.current_environ.name - if environ in {'clang', 'gnu'}: - self.build_system.cxxflags += ['-pthread'] - - @sanity_function - def assert_hello(self): - return sn.assert_found(r'Hello, World\!', self.stdout) diff --git a/tutorials/basics/hellomp/hellomp2.py b/tutorials/basics/hellomp/hellomp2.py deleted file mode 100644 index 01c7ddbe22..0000000000 --- a/tutorials/basics/hellomp/hellomp2.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class HelloThreadedExtendedTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - sourcepath = 'hello_threads.cpp' - build_system = 'SingleSource' - executable_opts = ['16'] - - @run_before('compile') - def set_compilation_flags(self): - self.build_system.cxxflags = ['-std=c++11', '-Wall'] - environ = self.current_environ.name - if environ in {'clang', 'gnu'}: - self.build_system.cxxflags += ['-pthread'] - - @sanity_function - def assert_num_messages(self): - num_messages = sn.len(sn.findall(r'\[\s?\d+\] Hello, World\!', - self.stdout)) - return sn.assert_eq(num_messages, 16) diff --git a/tutorials/basics/hellomp/hellomp3.py b/tutorials/basics/hellomp/hellomp3.py deleted file mode 100644 index 0719b65f90..0000000000 --- a/tutorials/basics/hellomp/hellomp3.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class HelloThreadedExtended2Test(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['*'] - sourcepath = 'hello_threads.cpp' - build_system = 'SingleSource' - executable_opts = ['16'] - - @run_before('compile') - def set_compilation_flags(self): - self.build_system.cppflags = ['-DSYNC_MESSAGES'] - self.build_system.cxxflags = ['-std=c++11', '-Wall'] - environ = self.current_environ.name - if environ in {'clang', 'gnu'}: - self.build_system.cxxflags += ['-pthread'] - - @sanity_function - def assert_num_messages(self): - num_messages = sn.len(sn.findall(r'\[\s?\d+\] Hello, World\!', - self.stdout)) - return sn.assert_eq(num_messages, 16) diff --git a/tutorials/basics/hellomp/src/hello_threads.cpp b/tutorials/basics/hellomp/src/hello_threads.cpp deleted file mode 100644 index 3ec541aade..0000000000 --- a/tutorials/basics/hellomp/src/hello_threads.cpp +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -// ReFrame Project Developers. See the top-level LICENSE file for details. -// -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include - - -#ifdef SYNC_MESSAGES -std::mutex hello_mutex; -#endif - - -void greetings(int tid) -{ -#ifdef SYNC_MESSAGES - const std::lock_guard lock(hello_mutex); -#endif - std::cout << "[" << std::setw(2) << tid << "] " << "Hello, World!\n"; -} - - -int main(int argc, char *argv[]) -{ - int nr_threads = 1; - if (argc > 1) { - nr_threads = std::atoi(argv[1]); - } - - if (nr_threads <= 0) { - std::cerr << "thread count must a be positive integer\n"; - return 1; - } - - std::vector threads; - for (auto i = 0; i < nr_threads; ++i) { - threads.push_back(std::thread(greetings, i)); - } - - for (auto &t : threads) { - t.join(); - } - - return 0; -} diff --git a/tutorials/basics/stream/stream1.py b/tutorials/basics/stream/stream1.py deleted file mode 100644 index 639b452bd9..0000000000 --- a/tutorials/basics/stream/stream1.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class StreamTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['gnu'] - prebuild_cmds = [ - 'wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c' # noqa: E501 - ] - build_system = 'SingleSource' - sourcepath = 'stream.c' - env_vars = { - 'OMP_NUM_THREADS': '4', - 'OMP_PLACES': 'cores' - } - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] - self.build_system.cflags = ['-fopenmp', '-O3', '-Wall'] - - @sanity_function - def validate_solution(self): - return sn.assert_found(r'Solution Validates', self.stdout) - - @performance_function('MB/s', perf_key='Copy') - def extract_copy_perf(self): - return sn.extractsingle(r'Copy:\s+(\S+)\s+.*', self.stdout, 1, float) - - @performance_function('MB/s', perf_key='Scale') - def extract_scale_perf(self): - return sn.extractsingle(r'Scale:\s+(\S+)\s+.*', self.stdout, 1, float) - - @performance_function('MB/s', perf_key='Add') - def extract_add_perf(self): - return sn.extractsingle(r'Add:\s+(\S+)\s+.*', self.stdout, 1, float) - - @performance_function('MB/s', perf_key='Triad') - def extract_triad_perf(self): - return sn.extractsingle(r'Triad:\s+(\S+)\s+.*', self.stdout, 1, float) diff --git a/tutorials/basics/stream/stream2.py b/tutorials/basics/stream/stream2.py deleted file mode 100644 index e1099820f9..0000000000 --- a/tutorials/basics/stream/stream2.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class StreamAltTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['gnu'] - prebuild_cmds = [ - 'wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c' # noqa: E501 - ] - build_system = 'SingleSource' - sourcepath = 'stream.c' - env_vars = { - 'OMP_NUM_THREADS': '4', - 'OMP_PLACES': 'cores' - } - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] - self.build_system.cflags = ['-fopenmp', '-O3', '-Wall'] - - @sanity_function - def validate_solution(self): - return sn.assert_found(r'Solution Validates', self.stdout) - - @performance_function('MB/s') - def extract_bw(self, kind='Copy'): - '''Generic performance extraction function.''' - - if kind not in ('Copy', 'Scale', 'Add', 'Triad'): - raise ValueError(f'illegal value in argument kind ({kind!r})') - - return sn.extractsingle(rf'{kind}:\s+(\S+)\s+.*', - self.stdout, 1, float) - - @run_before('performance') - def set_perf_variables(self): - '''Build the dictionary with all the performance variables.''' - - self.perf_variables = { - 'Copy': self.extract_bw(), - 'Scale': self.extract_bw('Scale'), - 'Add': self.extract_bw('Add'), - 'Triad': self.extract_bw('Triad'), - } diff --git a/tutorials/basics/stream/stream3.py b/tutorials/basics/stream/stream3.py deleted file mode 100644 index e8e3dc8954..0000000000 --- a/tutorials/basics/stream/stream3.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class StreamWithRefTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['gnu'] - prebuild_cmds = [ - 'wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c' # noqa: E501 - ] - build_system = 'SingleSource' - sourcepath = 'stream.c' - env_vars = { - 'OMP_NUM_THREADS': '4', - 'OMP_PLACES': 'cores' - } - reference = { - 'catalina': { - 'Copy': (25200, -0.05, 0.05, 'MB/s'), - 'Scale': (16800, -0.05, 0.05, 'MB/s'), - 'Add': (18500, -0.05, 0.05, 'MB/s'), - 'Triad': (18800, -0.05, 0.05, 'MB/s') - } - } - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] - self.build_system.cflags = ['-fopenmp', '-O3', '-Wall'] - - @sanity_function - def validate_solution(self): - return sn.assert_found(r'Solution Validates', self.stdout) - - @performance_function('MB/s') - def extract_bw(self, kind='Copy'): - '''Generic performance extraction function.''' - - if kind not in ('Copy', 'Scale', 'Add', 'Triad'): - raise ValueError(f'illegal value in argument kind ({kind!r})') - - return sn.extractsingle(rf'{kind}:\s+(\S+)\s+.*', - self.stdout, 1, float) - - @run_before('performance') - def set_perf_variables(self): - '''Build the dictionary with all the performance variables.''' - - self.perf_variables = { - 'Copy': self.extract_bw(), - 'Scale': self.extract_bw('Scale'), - 'Add': self.extract_bw('Add'), - 'Triad': self.extract_bw('Triad'), - } diff --git a/tutorials/basics/stream/stream4.py b/tutorials/basics/stream/stream4.py deleted file mode 100644 index 8a40a05f7b..0000000000 --- a/tutorials/basics/stream/stream4.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class StreamMultiSysTest(rfm.RegressionTest): - valid_systems = ['*'] - valid_prog_environs = ['cray', 'gnu', 'intel', 'nvidia'] - prebuild_cmds = [ - 'wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c' # noqa: E501 - ] - build_system = 'SingleSource' - sourcepath = 'stream.c' - env_vars = { - 'OMP_NUM_THREADS': 4, - 'OMP_PLACES': 'cores' - } - reference = { - 'catalina': { - 'Copy': (25200, -0.05, 0.05, 'MB/s'), - 'Scale': (16800, -0.05, 0.05, 'MB/s'), - 'Add': (18500, -0.05, 0.05, 'MB/s'), - 'Triad': (18800, -0.05, 0.05, 'MB/s') - } - } - - # Flags per programming environment - flags = variable(dict, value={ - 'cray': ['-fopenmp', '-O3', '-Wall'], - 'gnu': ['-fopenmp', '-O3', '-Wall'], - 'intel': ['-qopenmp', '-O3', '-Wall'], - 'nvidia': ['-mp', '-O3'] - }) - - # Number of cores for each system - cores = variable(dict, value={ - 'catalina:default': 4, - 'daint:gpu': 12, - 'daint:mc': 36, - 'daint:login': 10 - }) - - @run_before('compile') - def set_compiler_flags(self): - self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] - environ = self.current_environ.name - self.build_system.cflags = self.flags.get(environ, []) - - @run_before('run') - def set_num_threads(self): - num_threads = self.cores.get(self.current_partition.fullname, 1) - self.num_cpus_per_task = num_threads - self.env_vars = { - 'OMP_NUM_THREADS': num_threads, - 'OMP_PLACES': 'cores' - } - - @sanity_function - def validate_solution(self): - return sn.assert_found(r'Solution Validates', self.stdout) - - @performance_function('MB/s') - def extract_bw(self, kind='Copy'): - if kind not in {'Copy', 'Scale', 'Add', 'Triad'}: - raise ValueError(f'illegal value in argument kind ({kind!r})') - - return sn.extractsingle(rf'{kind}:\s+(\S+)\s+.*', - self.stdout, 1, float) - - @run_before('performance') - def set_perf_variables(self): - self.perf_variables = { - 'Copy': self.extract_bw(), - 'Scale': self.extract_bw('Scale'), - 'Add': self.extract_bw('Add'), - 'Triad': self.extract_bw('Triad'), - } diff --git a/tutorials/config/daint.py b/tutorials/config/daint.py deleted file mode 100644 index 8415b0fc2a..0000000000 --- a/tutorials/config/daint.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# and other ReFrame Project Developers. See the top-level LICENSE file for -# details. -# -# SPDX-License-Identifier: BSD-3-Clause - - -site_configuration = { - 'systems': [ - { - 'name': 'daint', - 'descr': 'Piz Daint Supercomputer', - 'hostnames': ['daint'], - 'modules_system': 'tmod32', - 'partitions': [ - { - 'name': 'login', - 'descr': 'Login nodes', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin', 'gnu', 'intel', 'nvidia', 'cray'], - }, - { - 'name': 'gpu', - 'descr': 'Hybrid nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C gpu', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100, - }, - { - 'name': 'mc', - 'descr': 'Multicore nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C mc', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100, - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'] - } - ] - } - ] - } - ], - 'environments': [ - { - 'name': 'gnu', - 'modules': ['PrgEnv-gnu'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'cray', - 'modules': ['PrgEnv-cray'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'intel', - 'modules': ['PrgEnv-intel'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'nvidia', - 'modules': ['PrgEnv-nvidia'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - } - ] # end of environments -} diff --git a/tutorials/config/daint_containers.py b/tutorials/config/daint_containers.py deleted file mode 100644 index f6174971cd..0000000000 --- a/tutorials/config/daint_containers.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# and other ReFrame Project Developers. See the top-level LICENSE file for -# details. -# -# SPDX-License-Identifier: BSD-3-Clause - - -site_configuration = { - 'systems': [ - { - 'name': 'daint', - 'descr': 'Piz Daint Supercomputer', - 'hostnames': ['daint'], - 'modules_system': 'tmod32', - 'partitions': [ - { - 'name': 'login', - 'descr': 'Login nodes', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin', 'gnu', 'intel', 'nvidia', 'cray'] - }, - # rfmdocstart: containers - { - 'name': 'gpu', - 'descr': 'Hybrid nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C gpu', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100, - 'container_platforms': [ - { - 'type': 'Sarus', - 'modules': ['sarus'] - }, - { - 'type': 'Singularity', - 'modules': ['singularity'] - } - ] - }, - # rfmdocend: containers - { - 'name': 'mc', - 'descr': 'Multicore nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C mc', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100 - } - ] - } - ] -} diff --git a/tutorials/config/daint_ext.py b/tutorials/config/daint_ext.py deleted file mode 100644 index 31d6d26f3f..0000000000 --- a/tutorials/config/daint_ext.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# and other ReFrame Project Developers. See the top-level LICENSE file for -# details. -# -# SPDX-License-Identifier: BSD-3-Clause - - -site_configuration = { - 'systems': [ - { - 'name': 'daint', - 'descr': 'Piz Daint Supercomputer', - 'hostnames': ['daint'], - 'modules_system': 'tmod32', - 'partitions': [ - { - 'name': 'login', - 'descr': 'Login nodes', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin', 'gnu', 'intel', 'nvidia', 'cray'] - }, - { - 'name': 'gpu', - 'descr': 'Hybrid nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C gpu', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100, - 'container_platforms': [ - { - 'type': 'Sarus', - 'modules': ['sarus'] - }, - { - 'type': 'Singularity', - 'modules': ['singularity'] - } - ], - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'] - } - ] - }, - { - 'name': 'mc', - 'descr': 'Multicore nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C mc', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100, - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'] - } - ] - } - ] - } - ], - 'environments': [ - { - 'name': 'gnu', - 'modules': ['PrgEnv-gnu'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'cray', - 'modules': ['PrgEnv-cray'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'intel', - 'modules': ['PrgEnv-intel'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'nvidia', - 'modules': ['PrgEnv-nvidia'], - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - }, - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': 'CC', - 'ftn': 'ftn', - 'target_systems': ['daint'] - } - ] -} diff --git a/tutorials/config/daint_mem.py b/tutorials/config/daint_mem.py deleted file mode 100644 index faed81ff2a..0000000000 --- a/tutorials/config/daint_mem.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# and other ReFrame Project Developers. See the top-level LICENSE file for -# details. -# -# SPDX-License-Identifier: BSD-3-Clause - - -site_configuration = { - 'systems': [ - { - 'name': 'daint', - 'descr': 'Piz Daint Supercomputer', - 'hostnames': ['daint'], - 'modules_system': 'tmod32', - 'partitions': [ - { - 'name': 'login', - 'descr': 'Login nodes', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin', 'gnu', 'intel', 'nvidia', 'cray'] - }, - # rfmdocstart: memory - { - 'name': 'gpu', - 'descr': 'Hybrid nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C gpu', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100, - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'] - } - ] - }, - # rfmdocend: memory - { - 'name': 'mc', - 'descr': 'Multicore nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-C mc', '-A csstaff'], - 'environs': ['gnu', 'intel', 'nvidia', 'cray'], - 'max_jobs': 100 - } - ] - } - ] -} diff --git a/tutorials/config/eum21.py b/tutorials/config/eum21.py deleted file mode 100644 index d882cb1f01..0000000000 --- a/tutorials/config/eum21.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Generic fallback configuration -# - -site_configuration = { - 'systems': [ - { - 'name': 'reframe-eum', - 'descr': 'HPC cluster for ReFrame tutorial at EUM', - 'hostnames': ['^reframe$'], - 'modules_system': 'lmod', - 'partitions': [ - { - 'name': 'login', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin', 'gnu'], - }, - { - 'name': 'cn', - 'descr': 'Compute nodes', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'access': ['-ptotal'], - 'environs': ['builtin', 'gnu', 'foss'], - 'max_jobs': 10, - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'] - } - ], - 'container_platforms': [ - { - 'type': 'Singularity', - } - ] - }, - ] - }, - { - 'name': 'generic', - 'descr': 'Generic example system', - 'hostnames': ['.*'], - 'partitions': [ - { - 'name': 'default', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['builtin'] - } - ] - }, - ], - 'environments': [ - { - 'name': 'gnu', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - 'modules': ['GCC/9.3.0'] - }, - { - 'name': 'foss', - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpif90', - 'modules': ['foss/2020a'] - }, - { - 'name': 'builtin', - 'cc': 'gcc', - 'cxx': 'g++', - 'ftn': 'gfortran', - 'target_systems': ['reframe-eum'] - }, - { - 'name': 'builtin', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - }, - ], - 'logging': [ - { - 'handlers': [ - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 - 'append': False - } - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': ( - '%(check_job_completion_time)s|reframe %(version)s|' - '%(check_info)s|jobid=%(check_jobid)s|' - '%(check_perf_var)s=%(check_perf_value)s|' - 'ref=%(check_perf_ref)s ' - '(l=%(check_perf_lower_thres)s, ' - 'u=%(check_perf_upper_thres)s)|' - '%(check_perf_unit)s' - ), - 'append': True - } - ] - } - ], -} diff --git a/tutorials/config/tresa.py b/tutorials/config/tresa.py deleted file mode 100644 index fb95298340..0000000000 --- a/tutorials/config/tresa.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# and other ReFrame Project Developers. See the top-level LICENSE file for -# details. -# -# SPDX-License-Identifier: BSD-3-Clause - - -site_configuration = { - 'systems': [ - { - 'name': 'tresa', - 'descr': 'My Mac', - 'hostnames': ['tresa'], - 'modules_system': 'nomod', - 'partitions': [ - { - 'name': 'default', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': ['gnu', 'clang'], - } - ] - } - ], - 'environments': [ - { - 'name': 'gnu', - 'cc': 'gcc-12', - 'cxx': 'g++-12', - 'ftn': 'gfortran-12', - 'target_systems': ['tresa'] - }, - { - 'name': 'clang', - 'cc': 'clang', - 'cxx': 'clang++', - 'ftn': '', - 'target_systems': ['tresa'] - }, - ] -} diff --git a/tutorials/deps/osu_benchmarks.py b/tutorials/deps/osu_benchmarks.py deleted file mode 100644 index da4f18e814..0000000000 --- a/tutorials/deps/osu_benchmarks.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import os - -import reframe as rfm -import reframe.utility.sanity as sn -import reframe.utility.udeps as udeps - - -# rfmdocstart: osupingpong -class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): - '''Base class of OSU benchmarks runtime tests''' - - valid_systems = ['daint:gpu'] - valid_prog_environs = ['gnu', 'nvidia', 'intel'] - sourcesdir = None - num_tasks = 2 - num_tasks_per_node = 1 - - @run_after('init') - def set_dependencies(self): - self.depends_on('OSUBuildTest', udeps.by_env) - - @sanity_function - def validate_test(self): - return sn.assert_found(r'^8', self.stdout) - - -@rfm.simple_test -class OSULatencyTest(OSUBenchmarkTestBase): - descr = 'OSU latency test' - - @require_deps - def set_executable(self, OSUBuildTest): - self.executable = os.path.join( - OSUBuildTest().stagedir, - 'mpi', 'pt2pt', 'osu_latency' - ) - self.executable_opts = ['-x', '100', '-i', '1000'] - - @performance_function('us') - def latency(self): - return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) -# rfmdocend: osupingpong - - -@rfm.simple_test -class OSUBandwidthTest(OSUBenchmarkTestBase): - descr = 'OSU bandwidth test' - - @require_deps - def set_executable(self, OSUBuildTest): - self.executable = os.path.join( - OSUBuildTest().stagedir, - 'mpi', 'pt2pt', 'osu_bw' - ) - self.executable_opts = ['-x', '100', '-i', '1000'] - - @performance_function('MB/s') - def bandwidth(self): - return sn.extractsingle(r'^4194304\s+(\S+)', - self.stdout, 1, float) - - -@rfm.simple_test -class OSUAllreduceTest(OSUBenchmarkTestBase): - mpi_tasks = parameter(1 << i for i in range(1, 5)) - descr = 'OSU Allreduce test' - - @run_after('init') - def set_num_tasks(self): - self.num_tasks = self.mpi_tasks - - @require_deps - def set_executable(self, OSUBuildTest): - self.executable = os.path.join( - OSUBuildTest().stagedir, - 'mpi', 'collective', 'osu_allreduce' - ) - self.executable_opts = ['-m', '8', '-x', '1000', '-i', '20000'] - - @performance_function('us') - def latency(self): - return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) - - -@rfm.simple_test -class OSUBuildTest(rfm.CompileOnlyRegressionTest): - descr = 'OSU benchmarks build test' - valid_systems = ['daint:gpu'] - valid_prog_environs = ['gnu', 'nvidia', 'intel'] - build_system = 'Autotools' - - @run_after('init') - def inject_dependencies(self): - self.depends_on('OSUDownloadTest', udeps.fully) - - @require_deps - def set_sourcedir(self, OSUDownloadTest): - self.sourcesdir = os.path.join( - OSUDownloadTest(part='login', environ='builtin').stagedir, - 'osu-micro-benchmarks-5.6.2' - ) - - @run_before('compile') - def set_build_system_attrs(self): - self.build_system.max_concurrency = 8 - - @sanity_function - def validate_build(self): - return sn.assert_not_found('error', self.stderr) - - -@rfm.simple_test -class OSUDownloadTest(rfm.RunOnlyRegressionTest): - descr = 'OSU benchmarks download sources' - valid_systems = ['daint:login'] - valid_prog_environs = ['builtin'] - executable = 'wget' - executable_opts = [ - 'http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.6.2.tar.gz' # noqa: E501 - ] - postrun_cmds = [ - 'tar xzf osu-micro-benchmarks-5.6.2.tar.gz' - ] - - @sanity_function - def validate_download(self): - return sn.assert_true(os.path.exists('osu-micro-benchmarks-5.6.2')) diff --git a/unittests/resources/config/settings.py b/unittests/resources/config/settings.py index 39f63b5e28..b888e0fbfc 100644 --- a/unittests/resources/config/settings.py +++ b/unittests/resources/config/settings.py @@ -61,6 +61,13 @@ def hostname(): '#DW jobdw capacity={capacity}', '#DW stage_in source={stagein_src}' ] + }, + { + 'name': 'uenv', + 'options': [ + '--mount={mount}', + '--file={file}' + ] } ], 'features': ['cuda', 'mpi'], diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 63209d0939..7fc8edd9fd 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -423,7 +423,8 @@ def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch): def test_performance_report(run_reframe, run_action): returncode, stdout, _ = run_reframe( checkpath=['unittests/resources/checks/frontend_checks.py'], - more_options=['-n', 'PerformanceFailureCheck', '--performance-report'], + more_options=['-n', '^PerformanceFailureCheck', + '--performance-report'], action=run_action ) if run_action == 'run': diff --git a/unittests/test_config.py b/unittests/test_config.py index 27c6f4e67f..0681c4645c 100644 --- a/unittests/test_config.py +++ b/unittests/test_config.py @@ -293,7 +293,7 @@ def test_select_subconfig(site_config): assert (site_config.get('systems/0/partitions/0/environs') == ['PrgEnv-gnu', 'builtin']) assert site_config.get('systems/0/partitions/0/descr') == 'GPU partition' - assert len(site_config.get('systems/0/partitions/0/resources')) == 2 + assert len(site_config.get('systems/0/partitions/0/resources')) == 3 assert (site_config.get('systems/0/partitions/0/resources/@gpu/name') == 'gpu') assert site_config.get('systems/0/partitions/0/modules') == [ @@ -461,6 +461,12 @@ def test_system_create(site_config): assert resources_spec == ['#DW jobdw capacity=100GB', '#DW stage_in source=/foo'] + env_resources_spec = partition.get_resource( + 'uenv', mount='mount_point', file='file_path' + ) + assert env_resources_spec == ['--mount=mount_point', + '--file=file_path'] + # Check processor info assert partition.processor.info is not None assert partition.processor.topology is not None diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 885973ca8d..04e0edb78c 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -1291,13 +1291,10 @@ def test_perf_logging_multiline(make_runner, make_exec_ctx, perf_test, config_perflog, tmp_path): make_exec_ctx( config_perflog( - fmt=( - '%(check_job_completion_time)s,%(version)s,' - '%(check_display_name)s,%(check_system)s,' - '%(check_partition)s,%(check_environ)s,' - '%(check_jobid)s,%(check_result)s,' - '%(check_perf_var)s=%(check_perf_value)s,%(check_perf_unit)s' - ), + fmt=('%(check_job_completion_time)s|reframe %(version)s|' + '%(check_name)s|%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|%(check_perf_unit)s'), logging_opts={'perflog_compat': True} ) ) @@ -1315,8 +1312,19 @@ def test_perf_logging_multiline(make_runner, make_exec_ctx, perf_test, # assert that the emitted lines are correct with open(logfile) as fp: lines = fp.readlines() - assert ',perf0=100.0,unit0' in lines[1] - assert ',perf1=50.0,unit1' in lines[2] + + version = osext.reframe_version() + assert lines[0] == ('job_completion_time|reframe version|name|' + 'perf_var=perf_value|ref=perf_ref ' + '(l=perf_lower_thres, u=perf_upper_thres)|perf_unit\n') + assert lines[1].endswith( + f'|reframe {version}|_MyPerfTest|' + f'perf0=100.0|ref=0 (l=null, u=null)|unit0\n' + ) + assert lines[2].endswith( + f'|reframe {version}|_MyPerfTest|' + f'perf1=50.0|ref=0 (l=null, u=null)|unit1\n' + ) def test_perf_logging_lazy(make_runner, make_exec_ctx, lazy_perf_test, @@ -1345,9 +1353,14 @@ def test_perf_logging_lazy(make_runner, make_exec_ctx, lazy_perf_test, assert os.path.exists(logfile) +@pytest.fixture(params=['%(check_result)s|%(check_#ALL)s', '%(check_#ALL)s']) +def perflog_fmt(request): + return request.param + + def test_perf_logging_all_attrs(make_runner, make_exec_ctx, perf_test, - config_perflog, tmp_path): - make_exec_ctx(config_perflog(fmt='%(check_result)s|%(check_#ALL)s')) + config_perflog, tmp_path, perflog_fmt): + make_exec_ctx(config_perflog(fmt=perflog_fmt)) logging.configure_logging(rt.runtime().site_config) runner = make_runner() testcases = executors.generate_testcases([perf_test]) @@ -1359,7 +1372,8 @@ def test_perf_logging_all_attrs(make_runner, make_exec_ctx, perf_test, header = fp.readline() loggable_attrs = type(perf_test).loggable_attrs() - assert len(header.split('|')) == len(loggable_attrs) + 1 + assert (len(header.split('|')) == + len(loggable_attrs) + (perflog_fmt != '%(check_#ALL)s')) def test_perf_logging_custom_vars(make_runner, make_exec_ctx, @@ -1400,3 +1414,28 @@ def test_perf_logging_param_test(make_runner, make_exec_ctx, perf_param_tests, 'default' / '_MyPerfParamTest.log') assert os.path.exists(logfile) assert _count_lines(logfile) == 3 + + +def test_perf_logging_sanity_failure(make_runner, make_exec_ctx, + config_perflog, tmp_path): + class _X(_MyPerfTest): + @sanity_function + def validate(self): + return sn.assert_true(0, msg='no way') + + make_exec_ctx(config_perflog( + fmt='%(check_result)s|%(check_fail_reason)s|%(check_perfvalues)s', + perffmt='%(check_perf_value)s|' + )) + logging.configure_logging(rt.runtime().site_config) + runner = make_runner() + testcases = executors.generate_testcases([_X()]) + _assert_no_logging_error(runner.runall, testcases) + + logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_X.log' + assert os.path.exists(logfile) + with open(logfile) as fp: + lines = fp.readlines() + + assert len(lines) == 2 + assert lines[1] == 'fail|sanity error: no way|None|None\n' diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index 851657acd0..93a40f1640 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -15,7 +15,7 @@ from reframe.core.backends import (getlauncher, getscheduler) from reframe.core.environments import Environment from reframe.core.exceptions import ( - ConfigError, JobError, JobNotStartedError, JobSchedulerError + ConfigError, JobError, JobNotStartedError, JobSchedulerError, SkipTestError ) from reframe.core.schedulers import Job from reframe.core.schedulers.slurm import _SlurmNode, _create_nodes @@ -124,7 +124,7 @@ def fake_job(make_job): def prepare_job(job, command='hostname', pre_run=None, post_run=None, - prepare_cmds=None): + prepare_cmds=None, strict_flex=True): environs = [Environment(name='foo', modules=['testmod_foo'])] pre_run = pre_run or ['echo prerun'] post_run = post_run or ['echo postrun'] @@ -137,7 +137,8 @@ def prepare_job(job, command='hostname', post_run ], environs, - prepare_cmds + prepare_cmds, + strict_flex, ) @@ -1169,11 +1170,16 @@ def test_flex_alloc_no_num_tasks_per_node(make_flexible_job): assert job.num_tasks == 1 -def test_flex_alloc_not_enough_idle_nodes(make_flexible_job): +@pytest.fixture(params=['skip', 'error']) +def strict_flex(request): + return request.param == 'error' + + +def test_flex_alloc_not_enough_idle_nodes(make_flexible_job, strict_flex): job = make_flexible_job('idle') job.num_tasks = -12 - with pytest.raises(JobError): - prepare_job(job) + with pytest.raises(JobError if strict_flex else SkipTestError): + prepare_job(job, strict_flex=strict_flex) def test_flex_alloc_maintenance_nodes(make_flexible_job):