diff --git a/examples/basic_gromacs_config.yaml b/examples/basic_gromacs_config.yaml index e22461d2a..cfd61d6b8 100644 --- a/examples/basic_gromacs_config.yaml +++ b/examples/basic_gromacs_config.yaml @@ -1,4 +1,7 @@ ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: processes_per_node: 16 mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' @@ -38,8 +41,8 @@ ramble: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 target=x86_64 - compiler_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 target=x86_64 + compiler_spec: gcc@9.4.0 impi2018: spack_spec: intel-mpi@2018.4.274 target=x86_64 compiler: gcc9 diff --git a/examples/vector_gromacs_software_config.yaml b/examples/vector_gromacs_software_config.yaml index 24a1daa80..2099c7053 100644 --- a/examples/vector_gromacs_software_config.yaml +++ b/examples/vector_gromacs_software_config.yaml @@ -1,4 +1,7 @@ ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: processes_per_node: 16 mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' @@ -12,7 +15,7 @@ ramble: '{type}_{n_ranks}ranks-{gromacs_version}': # Arbitrary experiment name variables: app_workloads: ['water_gmx50', 'water_bare'] - n_ranks: [1, 2, 4] + n_ranks: [1, 2] n_threads: '1' size: '0003' type: ['pme', 'rf'] @@ -26,8 +29,8 @@ ramble: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 target=x86_64 - compiler_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 target=x86_64 + compiler_spec: gcc@9.4.0 impi2018: spack_spec: intel-mpi@2018.4.274 target=x86_64 compiler: gcc9 diff --git a/examples/vector_matrix_gromacs_config.yaml b/examples/vector_matrix_gromacs_config.yaml index 7135d8b9f..bfa883aec 100644 --- a/examples/vector_matrix_gromacs_config.yaml +++ b/examples/vector_matrix_gromacs_config.yaml @@ -1,4 +1,7 @@ ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: processes_per_node: 16 mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' @@ -11,7 +14,7 @@ ramble: '{type}_{n_ranks}ranks': # Arbitrary experiment name variables: app_workloads: ['water_gmx50', 'water_bare'] - n_ranks: [1, 2, 4] + n_ranks: [1, 2] n_threads: '1' size: '0003' type: ['pme', 'rf'] @@ -23,8 +26,8 @@ ramble: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 target=x86_64 - compiler_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 target=x86_64 + compiler_spec: gcc@9.4.0 impi2018: spack_spec: intel-mpi@2018.4.274 target=x86_64 compiler: gcc9 diff --git a/lib/ramble/docs/getting_started.rst b/lib/ramble/docs/getting_started.rst index 4135999e4..073ed373b 100644 --- a/lib/ramble/docs/getting_started.rst +++ b/lib/ramble/docs/getting_started.rst @@ -195,6 +195,8 @@ that experiments can be generated from. The ``ramble info`` command can also be used to see what variables each workload has, and potentially some suggested values for variables with a limited set of allowed values. +.. _ramble-workspaces: + ------------------ Ramble Workspaces ------------------ diff --git a/lib/ramble/docs/tutorials/10_using_modifiers.rst b/lib/ramble/docs/tutorials/10_using_modifiers.rst index a0ffb26c0..d8e52b8c1 100644 --- a/lib/ramble/docs/tutorials/10_using_modifiers.rst +++ b/lib/ramble/docs/tutorials/10_using_modifiers.rst @@ -121,8 +121,11 @@ The resulting configuration file should look like the following. .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -135,12 +138,12 @@ The resulting configuration file should look like the following. experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -212,8 +215,11 @@ look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -227,12 +233,12 @@ look like the following: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -279,8 +285,11 @@ configuration file should look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -294,12 +303,12 @@ configuration file should look like the following: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 diff --git a/lib/ramble/docs/tutorials/11_using_internals.rst b/lib/ramble/docs/tutorials/11_using_internals.rst index 336ddecf6..718c40b5f 100644 --- a/lib/ramble/docs/tutorials/11_using_internals.rst +++ b/lib/ramble/docs/tutorials/11_using_internals.rst @@ -109,8 +109,11 @@ the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -133,12 +136,12 @@ the following: redirect: '{experiment_run_dir}/end_time' use_mpi: false variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -203,8 +206,11 @@ configuration file should look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -237,12 +243,12 @@ configuration file should look like the following: - execute - end_time variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 diff --git a/lib/ramble/docs/tutorials/1_hello_world.rst b/lib/ramble/docs/tutorials/1_hello_world.rst index 71cf979a2..30a2a289b 100644 --- a/lib/ramble/docs/tutorials/1_hello_world.rst +++ b/lib/ramble/docs/tutorials/1_hello_world.rst @@ -48,6 +48,19 @@ filter available application definitions. For example: might output the following: +.. code-block:: console + + ==> 10 applications + hmmer hostname hpcc hpcg hpl intel-hpl intel-mpi-benchmarks lulesh osu-micro-benchmarks ufs-weather-model + +The ``ramble list`` command also accepts regular expressions. For example: + +.. code-block:: console + + $ ramble list h* + +might output the following: + .. code-block:: console ==> 5 applications @@ -95,7 +108,7 @@ workspace. You can call this workspace ``hello_world``. $ ramble workspace create hello_world -This will create a workspace for you in: +This will create a named workspace for you in: .. code-block:: console @@ -111,6 +124,16 @@ Now you can activate the workspace and view its default configuration. You can use the ``ramble workspace info`` command after editing configuration files to see how ramble would use the changes you made. +As an aside, if you had used: + +.. code-block:: console + + $ ramble workspace create -d hello_world + +Ramble would create an anonymous workspace for you in ``${PWD}/hello_world`` +for more information on named and anonymous workspaces, see +:ref:`Ramble workspace documentation`. + Configure the Workspace ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -125,7 +148,11 @@ includes the software stack, the experiments, and all variables. * ``execute_experiment.tpl`` This file is a template shell script that will be rendered to execute each of the experiments that ramble generates. -You can edit these files directly or with the command ``ramble workspace edit``. +You can edit these files directly or with the command: + +.. code-block:: console + + $ ramble workspace edit To begin, you should edit the ``ramble.yaml`` file to set up the configuration for your experiments. For this tutorial, replace the default yaml text with the @@ -217,3 +244,19 @@ extracted figures of merit. If the experiments were successful, this file will show the following results: * possible hostname: hostname of machine the experiment was executed on + +Cleanup the Workspace +^^^^^^^^^^^^^^^^^^^^^ + +After you are finished exploring the workspace and tutorial content, make sure +you deactivate the workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace you used with: + +.. code-block:: console + + $ ramble workspace remove hello_world diff --git a/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst b/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst index 550b99f17..e88fead27 100644 --- a/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst +++ b/lib/ramble/docs/tutorials/2_running_a_simple_gromacs_experiment.rst @@ -24,11 +24,12 @@ Application Information ----------------------- As mentioned above, this tutorial uses the `GROMACS `_ -application definition. We will begin with the `water_bare` and `water_gmx50` +application definition. We will begin with the ``water_bare`` and ``water_gmx50`` workloads, as they are able to execute in a short amount of time. Using a previously installed ``ramble``, the following command can be used to get information about these workloads: + .. code-block:: console $ ramble info gromacs @@ -125,8 +126,13 @@ For each setup run, a set of logs will be created at: $ $ramble_root/var/ramble/workspaces/$workspace_root/logs -Each run will have its own primary log, along with a folder containing a log for each -experiment that is being configured. +Each run will have its own primary log, along with a folder containing a log +for each experiment that is being configured. While setup is running, you can +monitor the process by looking at the contents of: + +.. code-block:: console + + $ $ramble_root/var/ramble/workspaces/basic_gromacs/logs/setup.latest/gromacs.water_gmx50.pme_single_rank.out Executing Experiments --------------------- @@ -160,3 +166,37 @@ show the following results: * Nanosecs per day: Nanoseconds of simulation per day at the speed achieved * Hours per nanosec: Hours required to calculate 1 nanosecond of simulation at the speed achieved + +Workspace Directory Structure +----------------------------- + +After analyzing the workspace, you can exmine the structure of the workspace at: + +.. code-block:: console + + $ $ramble_root/var/ramble/workspaces/basic_gromacs + +Within this directory, you should see the following directories: + + * ``configs`` - Contains all of the configuration files for the workspace + * ``experiments`` - Contains all of the experiment execution directories + * ``inputs`` - Contains all of the input files needed for the experiments + * ``logs`` - Contains all log files from any Ramble command that acts on the workspace + * ``shared`` - Contains auxiliary files the experiment might need, such as environment variable information for licenses + * ``software`` - Contains software environments for the experiments + +Cleanup the Workspace +--------------------- + +Once you are finished with the tutorial content, ensure you have deactivated +the workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace using: + +.. code-block:: console + + $ ramble workspace remove basic_gromacs diff --git a/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst b/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst index 5f4892552..70026a266 100644 --- a/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst +++ b/lib/ramble/docs/tutorials/3_modifying_a_gromacs_experiment.rst @@ -113,9 +113,8 @@ The experiments section of this command's output might contain the following: type = rf ==> rf The variables ``mpi_command``, ``processes_per_node``, and ``batch_submit`` -come from the workspace scope on lines 3, 4, and 5 of the YAML file in -:ref:`gromacs_workspace_config`. Each experiment has its own definition for -``n_ranks``, ``n_threads``, ``size``, and ``type``. +come from the workspace scope towards the top of the YAML file. Each experiment +has its own definition for ``n_ranks``, ``n_threads``, ``size``, and ``type``. The experiments in this configuration use both the ``water_bare`` and ``water_gmx50`` workloads. @@ -176,7 +175,7 @@ your workspace's ``ramble.yaml`` file. While these variables contain ``Suggested Values`` some other variable definitions can take any value you want. As a result, they might not provide anything other than a default value. While the default value is expected to -function properly, you as the user can override their definition within the +function properly, you are allowed to override their definition within the ``ramble.yaml`` if your experiments would benefit from it. However, be aware that this also has the potential to change the behavior of your experiments and is considered an advanced action. @@ -187,7 +186,9 @@ Editing Experiments Now that you know how to determine which values are possible, select one or more of the possible values for the ``size`` variable. You will modify the experiments in your workspace to use this (remember, you have 4 experiments -defined currently). **NOTE** The larger the value, the more expensive the +defined currently). + +**NOTE** The larger the value, the more expensive the experiment will be. However you can give each experiment a unique value. To make editing the workspace easier, use the following command (assuming you @@ -217,6 +218,23 @@ These changes should now be reflected in the output of: .. include:: shared/gromacs_execute.rst -If you have a ``results`` file from the :ref:`running_an_experiment_tutorial`, +If you have a ``results`` file from the :ref:`previous tutorial `, you can compare it with the newly created results file to see what impact changing the ``size`` variable had on your figures of merit. + +Cleaning the Workspace +---------------------- + +After you are finished with the content of this tutorial, make sure you +deactivate your workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +If you no longer need the workspace materials, remove the entire workspace +with: + +.. code-block:: console + + $ ramble workspace remove basic_gromacs diff --git a/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst b/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst index bf53d7674..1b9495cd4 100644 --- a/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst +++ b/lib/ramble/docs/tutorials/4_using_vectors_and_matrices.rst @@ -232,7 +232,9 @@ expanding the ``{type}`` variable reference. **NOTE** Because we are editing YAML, the experiment name needs to be explicitly delimited as a string. Notice how in the above example we wrap the -experiment name in single quotes to explicitly make it a string. +experiment name in single quotes to explicitly make it a string. Without this, +YAML parsers identify the leading ``{`` character, and assume the content is a +dictionary. Now, save and exit the file. The resulting experiments can be seen using: @@ -396,14 +398,16 @@ Defining a Scaling Study The final modification you'll make to this workspace is to update the experiment definition to perform a basic rank based scaling study. -Edit the ``ramble.yaml`` file, and update the value for ``n_ranks`` to be -``[1, 2, 4]``. After doing this, add the ``n_ranks`` variable to the matrix -definition, and ensure your experiment name uses the ``{n_ranks}`` placeholder. +Edit the ``ramble.yaml`` file, and perform the following steps: + + #. Update the value for ``n_ranks`` to be ``[1, 2, 4]`` + #. Add the ``n_ranks`` variable to the matrix definition + #. Ensure your experiment name uses the ``{n_ranks}`` placeholder + At this point, your complete ``ramble.yaml`` file should look like the following: .. literalinclude:: ../../../../examples/vector_matrix_gromacs_config.yaml - :linenos: :language: YAML However, your experiment name template may look different from the above, as diff --git a/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst b/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst index 377ad0768..c71f17000 100644 --- a/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst +++ b/lib/ramble/docs/tutorials/5_changing_your_software_stack.rst @@ -20,7 +20,7 @@ for molecular dynamics. This tutorial builds off of concepts introduced in previous tutorials. Please make sure you review those before starting with this tutorial's content. -.. include:: shared/gromacs_workspace.rst +.. include:: shared/gromacs_vector_workspace.rst Software Description -------------------- @@ -74,11 +74,39 @@ a software environment named ``gromacs``. This can be overridden using the variable ``env_name``, which can be the name of any environment defined in the workspace configuration file. -The above output relates to lines 37-53 in the above configuration file, where -we define packages for Intel MPI, GROMACS, and GCC 9.3.0. The GROMACS and Intel -MPI packages are then combined into the ``gromacs`` software environment. Lines -40, 43, and 46 are the names of packages within this workspace, while line 50 -is the name of a software environment. + +The relevant portion of the workspace configuration file is: + +.. code-block:: YAML + + spack: + concretized: true + packages: + gcc9: + spack_spec: gcc@9.3.0 target=x86_64 + compiler_spec: gcc@9.3.0 + impi2018: + spack_spec: intel-mpi@2018.4.274 target=x86_64 + compiler: gcc9 + gromacs: + spack_spec: gromacs@2021.6 + compiler: gcc9 + environments: + gromacs: + packages: + - gromacs + - impi2018 + +In this configuration, the ``packages`` block defines software packages that +can be used to build experiment environments out of. The ``environments`` block +defines software environments which can be used for the experiments listed +within the ``ramble:applications`` block. Keys within both ``packages`` and +``environments`` are the name of the package or environment (respectively). +Each environment has a ``packages`` block, which contains a list of package +names that are defined in the higher level ``packages`` block. + +These are further documented in the +:ref:`Spack configuration section` documentation. Changing Software Definitions ----------------------------- @@ -159,8 +187,31 @@ should similarly be unique, and can use placeholder values for variable definitions. As an example, to explore both of the versions of GROMACS described in this -tutorial, your ``ramble.yaml`` coudld look like the following: +tutorial, your ``ramble.yaml`` could look like the following: .. literalinclude:: ../../../../examples/vector_gromacs_software_config.yaml - :linenos: :language: YAML + +Using this configuration file, you can examine what changes it would make to +your workspace through: + +.. code-block:: console + + $ ramble workspace info + or; + $ ramble workspace setup --dry-run + +Cleaning the Workspace +---------------------- + +Once your are finished with the tutorial content, deactivate your workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the entire workspace and all of its contents using: + +.. code-block:: console + + $ ramble workspace remove basic_gromacs diff --git a/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst b/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst index deeee0124..d691c3bfb 100644 --- a/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst +++ b/lib/ramble/docs/tutorials/6_configuring_a_scaling_study.rst @@ -90,8 +90,8 @@ only want to change the ``n_nodes`` variable definition in our scaling study, and as a result the experiment name template will only include this template parameter. However, you are free to add additional parameters based on the experiments you would like to perform. We will also assume ``n_nodes`` will -take the values of ``1``, ``2``, and ``4``, however you should edit this for -the system you are attempting to run these experiments on. The contents of the +take the values of ``1`` and ``2``, however you should edit this for the system +you are attempting to run these experiments on. The contents of the configuration file might look like the following now: .. code-block:: YAML @@ -104,7 +104,7 @@ configuration file might look like the following now: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] At this point, you can attempt to view the experiments defined by this configuration file. To do this, use the following command: @@ -161,8 +161,11 @@ following after adding this information: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: 'mpirun -n {n_ranks}' @@ -173,7 +176,7 @@ following after adding this information: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] **NOTE** The value of the ``n_ranks`` variable is escaped using single quotes. This is because YAML interprets the ``{`` character as beginning a dictionary, @@ -199,8 +202,11 @@ look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -211,7 +217,7 @@ look like the following: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: diff --git a/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst b/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst index cc4f86b71..b7743194a 100644 --- a/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst +++ b/lib/ramble/docs/tutorials/7_using_zips_and_matrices.rst @@ -46,7 +46,8 @@ active). .. include:: shared/wrf_scaling_workspace.rst We will now expand this to perform more experiments using the zip -(:ref:`ramble-explicit-zips`) and matrix (:ref:`ramble-matrix-logic`) +(:ref:`zips documentation`) and matrix +(:ref:`matrix documentation`) functionality in Ramble. Construct Platforms Zip @@ -65,7 +66,7 @@ length. As an example, imagine we had the following variable / zip definitions: variables: platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + processes_per_node: ['16', '20'] zips: platform_config: - platform @@ -85,12 +86,15 @@ the above section. The result should look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} - platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + platform: ['platform1', 'platform2', 'platform3'] + processes_per_node: ['16', '18', '20'] zips: platform_config: - platform @@ -102,12 +106,12 @@ the above section. The result should look like the following: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -127,10 +131,10 @@ Define an Experiment Matrix At this point, your workspace should have a single zip along with one other list variable, ``n_nodes``. This configuration will not work properly for two reasons. -The first is that neither the zip, nor ``n_nodes`` are unconsumed. Unconsumed -zips and variables are zipped together to attempt to create a set of -experiments. In this case, ``n_nodes`` is a different length than -``platform_config``, and Ramble will refuse to zip them. Running: +The first is that neither the zip ( ``platform_config`` ), nor ``n_nodes`` are +unconsumed. Unconsumed zips and variables are zipped together to attempt to +create a set of experiments. In this case, ``n_nodes`` is a different length +than ``platform_config``, and Ramble will refuse to zip them. Running: .. code-block:: console @@ -141,9 +145,9 @@ might give the following error messages: .. code-block:: console ==> Error: Length mismatch in vector variables in experiment scaling_{n_nodes} - Variable n_nodes has length 3 - Variable platform has length 2 - Variable processes_per_node has length 2 + Variable n_nodes has length 2 + Variable platform has length 3 + Variable processes_per_node has length 3 This error message identifies that the ``platform_config`` zip was unzipped (since it is not consumed) and the length of the resulting variables are @@ -170,12 +174,15 @@ should have the following in your ``ramble.yaml``: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} - platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + platform: ['platform1', 'platform2', 'platform3'] + processes_per_node: ['16', '18', '20'] zips: platform_config: - platform @@ -187,7 +194,7 @@ should have the following in your ``ramble.yaml``: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] matrix: - platform_config - n_nodes @@ -195,7 +202,7 @@ should have the following in your ``ramble.yaml``: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -230,12 +237,15 @@ Your final configuration file should look something like: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} - platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + platform: ['platform1', 'platform2', 'platform3'] + processes_per_node: ['16', '18', '20'] zips: platform_config: - platform @@ -247,7 +257,7 @@ Your final configuration file should look something like: experiments: scaling_{n_nodes}_{platform}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] matrix: - platform_config - n_nodes @@ -255,7 +265,7 @@ Your final configuration file should look something like: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -271,6 +281,31 @@ Your final configuration file should look something like: .. include:: shared/wrf_execute.rst +**NOTE** Some of these experiments can take a while to execute. Experiments can +be filtered using the :ref:`--where`` option to execute the +higher scale experiments if desired. To do this, try: + +.. code-block:: console + + $ ramble workspace setup --where '{n_ranks} >= 20' + $ ramble on + $ ramble workspace analyze --where '{n_ranks} >= 20' + Ramble also supports generating machine readable results in YAML or JSON format. To use this functionality, use the ``--formats`` option on ``ramble workspace analyze``. + +Clean the Worksapce +------------------- + +Once you are finished with the tutorial content, make sure you deactivate the workspace using: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace and all of its content with: + +.. code-block:: console + + $ ramble workspace remove zips_and_matrices_wrf diff --git a/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst b/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst index 3b5f098dc..0d550cc6b 100644 --- a/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst +++ b/lib/ramble/docs/tutorials/8_var_expansion_indirection_and_stack_parameterization.rst @@ -66,12 +66,15 @@ final configuration from the previous tutorial. .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + processes_per_node: ['16', '18'] zips: platform_config: - platform @@ -83,7 +86,7 @@ final configuration from the previous tutorial. experiments: scaling_{n_nodes}_{platform}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] matrix: - platform_config - n_nodes @@ -91,7 +94,7 @@ final configuration from the previous tutorial. concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -129,7 +132,7 @@ like the following: packages: openmpi: - spack_spec: openmpi@4.1.6 + spack_spec: openmpi@3.1.6 +orterunprefix In the definition of the Intel MPI package above, you'll see we originally specified a ``compiler`` attribute (with the value of ``gcc9``). This can be @@ -153,19 +156,22 @@ parameterize the generation of these using a new variable definition. mpi_name: ['intel-mpi', 'openmpi'] Will create two software environments. One named ``wrfv4-intel-mpi`` and -another named ``wrfv4-openmpi``. The definition of ``mpi_name`` can be hoisted -to the workspace level, however we need to include it in the experiment +another named ``wrfv4-openmpi``. However, the definition of ``mpi_name`` can be +hoisted to the workspace level because we need to include it in the experiment generation as well. The result might look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + processes_per_node: ['16', '18'] mpi_name: ['intel-mpi', 'openmpi'] zips: platform_config: @@ -178,7 +184,7 @@ generation as well. The result might look like the following: experiments: scaling_{n_nodes}_{platform}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] matrix: - platform_config - n_nodes @@ -186,12 +192,12 @@ generation as well. The result might look like the following: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 openmpi: - spack_spec: openmpi@4.1.6 + spack_spec: openmpi@3.1.6 +orterunprefix wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf @@ -227,12 +233,15 @@ into the matrix. The result might look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + processes_per_node: ['16', '18'] mpi_name: ['intel-mpi', 'openmpi'] zips: platform_config: @@ -245,7 +254,7 @@ into the matrix. The result might look like the following: experiments: scaling_{n_nodes}_{platform}_{mpi_name}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] matrix: - platform_config - n_nodes @@ -254,12 +263,12 @@ into the matrix. The result might look like the following: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 openmpi: - spack_spec: openmpi@4.1.6 + spack_spec: openmpi@3.1.6 +orterunprefix wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf @@ -316,8 +325,8 @@ which variables you want to use. For example: .. code-block:: YAML variables: - intel-mpi_args: '--np {n_ranks} --map-by ppr:{processes_per_node}:node' - openmpi_args: '-n {n_ranks} -ppn {processes_per_node}' + openmpi_args: '--np {n_ranks} --map-by ppr:{processes_per_node}:node -x OMP_NUM_THREADS' + intel-mpi_args: '-n {n_ranks} -ppn {processes_per_node}' mpi_command: 'mpirun {{mpi_name}_args}' Allows the ``mpi_command`` definition to change based on the definition of @@ -328,10 +337,13 @@ resulting configuration might look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + processes_per_node: ['16', '18'] # Execution Template batch_submit: '{execute_experiment}' @@ -340,7 +352,7 @@ resulting configuration might look like the following: # Experiment Expansions mpi_name: ['intel-mpi', 'openmpi'] intel-mpi_args: '-n {n_ranks} -ppn {processes_per_node}' - openmpi_args: '--np {n_ranks} --map-by ppr:{processes_per_node}:node' + openmpi_args: '--np {n_ranks} --map-by ppr:{processes_per_node}:node -x OMP_NUM_THREADS' zips: platform_config: - platform @@ -352,7 +364,7 @@ resulting configuration might look like the following: experiments: scaling_{n_nodes}_{platform}_{mpi_name}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] matrix: - platform_config - n_nodes @@ -361,12 +373,12 @@ resulting configuration might look like the following: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 openmpi: - spack_spec: openmpi@4.1.6 + spack_spec: openmpi@3.1.6 +orterunprefix wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf @@ -377,6 +389,10 @@ resulting configuration might look like the following: - '{mpi_name}' - wrfv4 +**NOTE** The arguments for the various MPI implementations may not run on your +system if you require additional arguments. To be able to execute these on your +system, make sure you modify these appropriately. + At this point, you have described the 12 experiments you want to run, however they are still not completely defined. Running: @@ -410,10 +426,13 @@ The resulting configuration file might look like the following: .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: n_ranks: '{processes_per_node}*{n_nodes}' platform: ['platform1', 'platform2'] - processes_per_node: ['2', '4'] + processes_per_node: ['16', '18'] # Execution Template batch_submit: '{execute_experiment}' @@ -422,7 +441,7 @@ The resulting configuration file might look like the following: # Experiment Expansions mpi_name: ['intel-mpi', 'openmpi'] intel-mpi_args: '-n {n_ranks} -ppn {processes_per_node}' - openmpi_args: '--np {n_ranks} --map-by ppr:{processes_per_node}:node' + openmpi_args: '--np {n_ranks} --map-by ppr:{processes_per_node}:node -x OMP_NUM_THREADS' zips: platform_config: - platform @@ -434,7 +453,7 @@ The resulting configuration file might look like the following: experiments: scaling_{n_nodes}_{platform}_{mpi_name}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] env_name: 'wrfv4-{mpi_name}' matrix: - platform_config @@ -444,12 +463,12 @@ The resulting configuration file might look like the following: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 openmpi: - spack_spec: openmpi@4.1.6 + spack_spec: openmpi@3.1.6 +orterunprefix wrfv4: spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf @@ -477,3 +496,18 @@ experiment directories. Looking at these, you should see the correct MPI arguments within the relevant experiments. .. include:: shared/wrf_execute.rst + +Clean the Workspace +------------------- + +Once you are finished with the tutorial content, make sure you deactivate your workspace: + +.. code-block:: console + + $ ramble workspace deactivate + +Additionally, you can remove the workspace and all of its content with: + +.. code-block:: console + + $ ramble workspace remove var_expansion_and_indirection diff --git a/lib/ramble/docs/tutorials/9_success_criteria.rst b/lib/ramble/docs/tutorials/9_success_criteria.rst index 710ca73d5..29a378524 100644 --- a/lib/ramble/docs/tutorials/9_success_criteria.rst +++ b/lib/ramble/docs/tutorials/9_success_criteria.rst @@ -87,6 +87,7 @@ This success criteria might look like the following: - name: 'timing-present' mode: 'string' match: 'Timing for main.*' + file: '{experiment_run_dir}/rsl.out.0000' Edit your workspace configuration file with: @@ -101,7 +102,7 @@ configuration file might look like the following: ramble: variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -111,17 +112,18 @@ configuration file might look like the following: - name: 'timing-present' mode: 'string' match: 'Timing for main.*' + file: '{experiment_run_dir}/rsl.out.0000' workloads: CONUS_12km: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -170,7 +172,7 @@ resulting configuration file might look like the following: ramble: variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -180,6 +182,7 @@ resulting configuration file might look like the following: - name: 'timing-present' mode: 'string' match: 'Timing for main.*' + file: '{experiment_run_dir}/rsl.out.0000' workloads: CONUS_12km: success_criteria: @@ -190,12 +193,12 @@ resulting configuration file might look like the following: experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -238,8 +241,16 @@ explore using this after you perform experiments. .. include:: shared/wrf_execute.rst To ensure the success criteria are checked and the experiments pass them, -ensure ``SUCCESS`` is printed for the status of each experiment. Also, running -analyze in debug mode as: +ensure ``SUCCESS`` is printed for the status of each experiment. + +When an experiment fails, you can force Ramble to print the figure of merit +data using: + +.. code-block:: console + + $ ramble workspace analyze --always-print-foms + +Also, running analyze in debug mode as: .. code-block:: console diff --git a/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst b/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst new file mode 100644 index 000000000..fe8dc6592 --- /dev/null +++ b/lib/ramble/docs/tutorials/shared/gromacs_vector_workspace.rst @@ -0,0 +1,76 @@ +.. Copyright 2022-2023 Google LLC + + Licensed under the Apache License, Version 2.0 or the MIT license + , at your + option. This file may not be copied, modified, or distributed + except according to those terms. + +Configuring experiments +------------------------ + +For this tutorial, you are going to focus on creating experiments from the +``water_bare`` and ``water_gmx50`` workloads. The default configuration will +contain experiments for each value of the ``type`` variable, and a single value +for the ``size`` variable. + +You will use a Ramble workspace to manage these experiments. + +Create and Activate a Workspace +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before you can configure your GROMACS experiments, you'll need to set up a +workspace. You can call this workspace ``basic_gromacs``. + +.. code-block:: console + + $ ramble workspace create basic_gromacs + +This will create a workspace for you in: + +.. code-block:: console + + $ $ramble_root/var/ramble/workspaces/basic_gromacs + +Now you can activate the workspace and view its default configuration. + +.. code-block:: console + + $ ramble workspace activate basic_gromacs + $ ramble workspace info + +You can use the ``ramble workspace info`` command after editing configuration +files to see how ramble would use the changes you made. + +Configure the Workspace +~~~~~~~~~~~~~~~~~~~~~~~ + +Within the workspace directory, ramble creates a directory named ``configs``. +This directory contains generated configuration and template files. Each of +these files can be edited to configure the workspace, and examples will be +provided below. + +The available files are: + + * ``ramble.yaml`` This file describes all aspects of the workspace. This includes the software stack, the experiments, and all variables. + * ``execute_experiment.tpl`` This file is a template shell script that will be rendered to execute each of the experiments that ramble generates. + +You can edit these files directly or with the command ``ramble workspace edit``. + +To begin, you should edit the ``ramble.yaml`` file to set up the configuration +for your experiments. For this tutorial, replace the default yaml text with the +contents of ``$ramble_root/examples/vector_matrix_gromacs_config.yaml``: + + +.. literalinclude:: ../../../../examples/vector_matrix_gromacs_config.yaml + :language: YAML + +Note that specifying compilers that Spack doesn't have installed may take a while. +To see available compilers, use ``spack compilers`` or see `Spack documentation +`_ +for more information. + +The second file you should edit is the ``execute_experiment.tpl`` template file. +This file contains a template script that will be rendered into an execution +script for each generated experiment. You can feel free to edit it as you need +to for your given system, but for this tutorial the default value will work. diff --git a/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst b/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst index 8a934032e..92efe61ab 100644 --- a/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst +++ b/lib/ramble/docs/tutorials/shared/gromacs_workspace.rst @@ -51,10 +51,9 @@ these files can be edited to configure the workspace, and examples will be provided below. The available files are: -* ``ramble.yaml`` This file describes all aspects of the workspace. This -includes the software stack, the experiments, and all variables. -* ``execute_experiment.tpl`` This file is a template shell script that will be -rendered to execute each of the experiments that ramble generates. + + * ``ramble.yaml`` This file describes all aspects of the workspace. This includes the software stack, the experiments, and all variables. + * ``execute_experiment.tpl`` This file is a template shell script that will be rendered to execute each of the experiments that ramble generates. You can edit these files directly or with the command ``ramble workspace edit``. @@ -64,7 +63,6 @@ contents of ``$ramble_root/examples/basic_gromacs_config.yaml``: .. literalinclude:: ../../../../examples/basic_gromacs_config.yaml - :linenos: :language: YAML Note that specifying compilers that Spack doesn't have installed may take a while. diff --git a/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst b/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst index a497ab8df..fa528b3fd 100644 --- a/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst +++ b/lib/ramble/docs/tutorials/shared/wrf_scaling_workspace.rst @@ -16,7 +16,7 @@ location can be seen under the ``Location`` output of: $ ramble workspace info -Additionally, the files can be edited directly with: +Alternatively, the files can be edited directly with: .. code-block:: console @@ -28,8 +28,11 @@ final configuration from a previous tutorial. .. code-block:: YAML ramble: + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' variables: - processes_per_node: 4 + processes_per_node: 16 n_ranks: '{processes_per_node}*{n_nodes}' batch_submit: '{execute_experiment}' mpi_command: mpirun -n {n_ranks} @@ -40,12 +43,12 @@ final configuration from a previous tutorial. experiments: scaling_{n_nodes}: variables: - n_nodes: [1, 2, 4] + n_nodes: [1, 2] spack: concretized: true packages: gcc9: - spack_spec: gcc@9.3.0 + spack_spec: gcc@9.4.0 intel-mpi: spack_spec: intel-mpi@2018.4.274 compiler: gcc9 @@ -59,8 +62,10 @@ final configuration from a previous tutorial. - intel-mpi - wrfv4 -The above configuration will execute 3 experiments, comprising a basic scaling -study on three different sets of nodes. This is primarily defined by the use of -vector experiments, which are documented at :ref:`ramble-vector-logic`. Vector -experiments were also introduced in :ref:`vector_and_matrix_tutorial`. +The above configuration will execute 2 experiments, comprising a basic scaling +study on 2 different sets of nodes. This is primarily defined by the use of +vector experiments, which are documented in the :ref:`vector +logic` portion of the workspace configuration file +documentation.. Vector experiments were also introduced in the :ref:`vector and +matrix tutorial `.