From 9589ecf526a9d0a6fae60453a297c6d8e6287890 Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Mon, 10 Feb 2025 20:35:04 +0200 Subject: [PATCH 01/10] advisor: Update advisor with oneAPI 2025 --- benchmarks/user/advisor/README.md | 41 +++-- benchmarks/user/advisor/roofline.py | 31 ++-- benchmarks/user/advisor/run_advisor.py | 37 +++-- devito/arch/compiler.py | 14 ++ devito/operator/profiling.py | 18 +-- .../performance/02_advisor_roofline.ipynb | 148 ++++-------------- 6 files changed, 116 insertions(+), 173 deletions(-) diff --git a/benchmarks/user/advisor/README.md b/benchmarks/user/advisor/README.md index 6627084996..a78f475f78 100644 --- a/benchmarks/user/advisor/README.md +++ b/benchmarks/user/advisor/README.md @@ -2,35 +2,50 @@ Example runs: * `python3 run_advisor.py --name isotropic --path /examples/seismic/acoustic/acoustic_example.py` * `python3 run_advisor.py --name tti_so8 --path /examples/seismic/tti/tti_example.py --exec-args "-so 8"` -* `python3 run_advisor.py --name iso_ac_so6 --path /benchmarks/user/benchmark.py --exec-args "bench -P acoustic -so 6 --tn 200 -d 100 100 100 --autotune off -x 1"` +* `python3 run_advisor.py --name iso_ac_so4 --path /benchmarks/user/benchmark.py --exec-args "run -P acoustic -so 4 --tn 200 -d 100 100 100"` -After the run has finished you should be able to plot a roofline with the results and export roofline data to JSON using: +After the run has finished you should be able to save a .json and plot the +roofline with the results: * `python3 roofline.py --name Roofline --project ` To create a read-only snapshot for use with Intel Advisor GUI, use: * `advixe-cl --snapshot --project-dir= pack -- /` Prerequisites: -* Support guaranteed only for Intel Advisor as installed with Intel Parallel Studio v 2020 Update 2 - and Intel oneAPI 2021; earlier years may not work; other 2020/2021 versions, as well as later years, - may or may not work. +* Support is guaranteed only for Intel oneAPI 2025; earlier versions may not work. +You may download Intel oneAPI [here](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html?packages=oneapi-toolkit&oneapi-toolkit-os=linux&oneapi-lin=apt). + +* Add Advisor (advixe-cl) and compilers (icx) in the path. It should be along the lines of: +```sh +source /opt/intel/oneapi/advisor/latest/env/vars.sh +source /opt/intel/oneapi/compiler/latest/env/vars.sh +``` +depending on where you installed oneAPI + * In Linux systems you may need to enable system-wide profiling by setting: - - `/proc/sys/kernel/yama/ptrace_scope` to `0` - - `/proc/sys/kernel/perf_event_paranoid` to `1` -* `numactl` must be available on the system. If not available, install with: - `sudo apt-get install numactl` +```sh +/proc/sys/kernel/yama/ptrace_scope to 0 +/proc/sys/kernel/perf_event_paranoid to 1 +``` + +* `numactl` must be available on the system. If not available, install using: +```sh +sudo apt-get install numactl +``` * Install `pandas` and `matplotlib`. They are not included in the core Devito installation. +```sh +pip install pandas matplotlib +``` Limitations: * Untested with more complicated examples. -* Untested on Intel KNL (we might need to ask `numactl` to bind to MCDRAM). -* Running the `tripcounts` analysis takes a lot, despite starting in paused +* Running the `tripcounts` analysis is time-consuming, despite starting in paused mode. This analysis, together with the `survey` analysis, is necessary to generate a roofline. Both are run by `run_advisor.py`. -* Requires python3, untested in earlier versions of python and conda environments -* Currently requires download of repository and running `pip3 install .`, the scripts +* Requires Python3, untested in conda environments +* Currently requires download of repository and running `pip install .`, the scripts are currently not included as a package with the user installation of Devito TODO: diff --git a/benchmarks/user/advisor/roofline.py b/benchmarks/user/advisor/roofline.py index ff208b14b2..852f376a54 100644 --- a/benchmarks/user/advisor/roofline.py +++ b/benchmarks/user/advisor/roofline.py @@ -18,7 +18,7 @@ import sys import os -from benchmarks.user.advisor.advisor_logging import check, err, log +from advisor_logging import check, err, log try: @@ -26,15 +26,14 @@ except ImportError: check(False, 'Error: Intel Advisor could not be found on the system,' ' make sure to source environment variables properly. Information can be' - ' found at https://software.intel.com/content/www/us/en/develop/' - 'documentation/advisor-user-guide/top/launch-the-intel-advisor/' - 'intel-advisor-cli/setting-and-using-intel-advisor-environment-variables.html') + ' found at https://www.intel.com/content/www/us/en/docs/advisor/' + 'user-guide/2024-2/set-up-environment-variables.html') sys.exit(1) matplotlib.use('Agg') # Use fancy plot colors -plt.style.use('seaborn-darkgrid') +plt.style.use('ggplot') @click.command() @@ -65,21 +64,25 @@ def roofline(name, project, scale, precision, mode, th): pd.options.display.max_rows = 20 - log('Opening project...') + log(f'Opening project {project}...') project = advisor.open_project(str(project)) if not project: - err('Could not open project %s.' % project) + err(f'Could not open project {project}.') log('Loading data...') data = project.load(advisor.SURVEY) rows = [{col: row[col] for col in row} for row in data.bottomup] roofs = data.get_roofs() + # Following deprecation solution from here: + # https://github.com/pandas-dev/pandas/issues/57734 + pd.set_option('future.no_silent_downcasting', True) full_df = pd.DataFrame(rows).replace('', np.nan) # Narrow down the columns to those of interest try: + analysis_columns = ['loop_name', 'self_ai', 'self_gflops', 'self_time'] df = full_df[analysis_columns].copy() except KeyError: err('Could not read data columns from profiling. Not enough data has been ' @@ -169,7 +172,7 @@ def roofline(name, project, scale, precision, mode, th): label_y = row.self_gflops ax.text(label_x, label_y, 'Time: %.2fs\n' - 'Incidence: %.0f%%' % (row.self_time, row.percent_weight), + 'Peak pct: %.0f%%' % (row.self_time, row.percent_weight), bbox={'boxstyle': 'round', 'facecolor': 'white'}, fontsize=8) top_loops_data = [{'ai': row.self_ai, 'gflops': row.self_gflops, @@ -198,19 +201,17 @@ def roofline(name, project, scale, precision, mode, th): legend = plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 7}, title='Rooflines') - # saving the chart in PNG format - plt.savefig('%s.png' % name, bbox_extra_artists=(legend,), bbox_inches='tight') + # saving the chart in PDF format + plt.savefig('%s.pdf' % name, bbox_extra_artists=(legend,), bbox_inches='tight') figpath = os.path.realpath(__file__).split(os.path.basename(__file__))[0] - log('Figure saved in %s%s.png.' % (figpath, name)) + log('\nFigure saved in %s%s.pdf.' % (figpath, name)) # Save the JSON file with open('%s.json' % name, 'w') as f: f.write(json.dumps(roofline_data)) - log('JSON file saved as %s.json.' % name) - - -analysis_columns = ['loop_name', 'self_ai', 'self_gflops', 'self_time'] + log('\nJSON file saved as %s.json.' % name) + log('Done!') if __name__ == '__main__': roofline() diff --git a/benchmarks/user/advisor/run_advisor.py b/benchmarks/user/advisor/run_advisor.py index c162926f63..e37ca38417 100644 --- a/benchmarks/user/advisor/run_advisor.py +++ b/benchmarks/user/advisor/run_advisor.py @@ -1,16 +1,14 @@ +import click import datetime import logging import os +import sys + from pathlib import Path from subprocess import check_output, PIPE, Popen -import sys from tempfile import gettempdir, mkdtemp -import click - - -from benchmarks.user.advisor.advisor_logging import (check, log, progress, - log_process) +from advisor_logging import (check, log, progress, log_process) @click.command() @@ -49,21 +47,21 @@ def run_with_advisor(path, output, name, exec_args): output = Path(output).joinpath(name) output.mkdir(parents=True, exist_ok=True) - # Intel Advisor and Intel compilers must be available through either Intel Parallel - # Studio or Intel oneAPI (currently tested versions include IPS 2020 Update 2 and - # oneAPI 2021 beta08) + # Intel Advisor and Intel compilers must be available through either + # Intel oneAPI (tested with Intel oneAPI 2025.1) try: ret = check_output(['advixe-cl', '--version']).decode("utf-8") except FileNotFoundError: check(False, "Error: Couldn't detect `advixe-cl` to run Intel Advisor.") try: - ret = check_output(['icc', '--version']).decode("utf-8") + ret = check_output(['icx', '--version']).decode("utf-8") + log("Found icx version: %s" % ret.strip()) except FileNotFoundError: - check(False, "Error: Couldn't detect Intel Compiler (icc).") + check(False, "Error: Couldn't detect Intel Compiler (icx).") # All good, Intel compiler and advisor are available - os.environ['DEVITO_ARCH'] = 'intel' + os.environ['DEVITO_ARCH'] = 'icx' # Tell Devito to instrument the generated code for Advisor os.environ['DEVITO_PROFILING'] = 'advisor' @@ -73,7 +71,7 @@ def run_with_advisor(path, output, name, exec_args): if devito_logging is None: os.environ['DEVITO_LOGGING'] = 'WARNING' - with progress('Setting up multi-threading environment'): + with progress('Setting up multi-threading environment with OpenMP'): # Roofline analyses are recommended with threading enabled os.environ['DEVITO_LANGUAGE'] = 'openmp' @@ -90,14 +88,14 @@ def run_with_advisor(path, output, name, exec_args): # Prevent NumPy from using threads, which otherwise leads to a deadlock when # used in combination with Advisor. This issue has been described at: - # `software.intel.com/en-us/forums/intel-advisor-xe/topic/780506` + # `software.intel.com/en-us/forums/intel-advisor-xe/topic/780506` # Note: we should rather sniff the BLAS library used by NumPy, and set the # appropriate env var only os.environ['OPENBLAS_NUM_THREADS'] = '1' os.environ['MKL_NUM_THREADS'] = '1' # Note: `Numaexpr`, used by NumPy, also employs threading, so we shall disable # it too via the corresponding env var. See: - # `stackoverflow.com/questions/17053671/python-how-do-you-stop-numpy-from-multithreading` # noqa + # `stackoverflow.com/questions/17053671/python-how-do-you-stop-numpy-from-multithreading` # noqa os.environ['NUMEXPR_NUM_THREADS'] = '1' # To build a roofline with Advisor, we need to run two analyses back to @@ -130,8 +128,8 @@ def run_with_advisor(path, output, name, exec_args): ] py_cmd = [sys.executable, str(path)] + exec_args.split() - # Before collecting the `survey` and `tripcounts` a "pure" python run to warmup the - # jit cache is preceded + # Before collecting the `survey` and `tripcounts` a "pure" python run + # to warmup the jit cache is preceded log('Starting Intel Advisor\'s `roofline` analysis for `%s`' % name) dt = datetime.datetime.now() @@ -147,6 +145,9 @@ def run_with_advisor(path, output, name, exec_args): advixe_handler.setFormatter(advixe_formatter) advixe_logger.addHandler(advixe_handler) + log("Project folder: %s" % str(output)) + log("Logging progress in: `%s`" % str(advixe_handler)) + with progress('Performing `cache warm-up` run'): try: p_warm_up = Popen(py_cmd, stdout=PIPE, stderr=PIPE) @@ -175,6 +176,8 @@ def run_with_advisor(path, output, name, exec_args): log('python3 roofline.py --name %s --project %s --scale %f' % (name, str(output), n_sockets)) + log('\nTo open the roofline using advixe-gui: ') + log('advixe-gui %s' % str(output)) if __name__ == '__main__': run_with_advisor() diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index aa8cfa0795..613f11c03d 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -769,6 +769,20 @@ def __init_finalize__(self, **kwargs): self.__init_intel_mpi__() self.__init_intel_mpi_flags__() + if configuration['profiling'] == 'advisor': + # Locate the Intel Advisor installation and + # add the necessary flags to the compiler + from devito.operator.profiling import locate_intel_advisor + path = locate_intel_advisor() + self.add_include_dirs(path.joinpath('include').as_posix()) + + _default_libs = ['ittnotify'] + self.add_libraries(_default_libs) + + libdir = path.joinpath('lib64').as_posix() + self.add_library_dirs(libdir) + self.add_ldflags('-Wl,-rpath,%s' % libdir) + def __init_intel_mpi__(self, **kwargs): # Make sure the MPI compiler uses an Intel compiler underneath, # whatever the MPI distro is diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index 32742489cd..63b671d413 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -353,7 +353,7 @@ class AdvancedProfilerVerbose2(AdvancedProfilerVerbose): class AdvisorProfiler(AdvancedProfiler): """ - Rely on Intel Advisor ``v >= 2020`` for performance profiling. + Rely on Intel Advisor 2025.1 for performance profiling. Tested versions of Intel Advisor: - As contained in Intel Parallel Studio 2020 v 2020 Update 2 - As contained in Intel oneAPI 2021 beta08 @@ -367,19 +367,13 @@ class AdvisorProfiler(AdvancedProfiler): _ext_calls = [_api_resume, _api_pause] def __init__(self, name): - self.path = locate_intel_advisor() - if self.path is None: + path = locate_intel_advisor() + if path is None: self.initialized = False else: - super().__init__(name) - # Make sure future compilations will get the proper header and - # shared object files - compiler = configuration['compiler'] - compiler.add_include_dirs(self.path.joinpath('include').as_posix()) - compiler.add_libraries(self._default_libs) - libdir = self.path.joinpath('lib64').as_posix() - compiler.add_library_dirs(libdir) - compiler.add_ldflags('-Wl,-rpath,%s' % libdir) + super(AdvisorProfiler, self).__init__(name) + + return def analyze(self, iet): return diff --git a/examples/performance/02_advisor_roofline.ipynb b/examples/performance/02_advisor_roofline.ipynb index 1c8d82a97a..e73f839a5c 100644 --- a/examples/performance/02_advisor_roofline.ipynb +++ b/examples/performance/02_advisor_roofline.ipynb @@ -6,13 +6,8 @@ "source": [ "## Using scripts to perform Intel Advisor roofline profiling on Devito\n", "\n", - "This notebook uses the prewritten scripts `run_advisor.py`, `roofline.py` and `advisor_to_json.py` to show how you can easily profile a Devito application using Intel Advisor 2020 and plot memory-bound/compute-bound roofline models depicting the current state of the application's performance. These scripts can be found in the `benchmarks/user/advisor` folder of the full Devito repository. They are also available as part of the Devito package." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ + "This notebook uses the prewritten scripts `run_advisor.py` and `roofline.py` to show how you can easily profile a Devito application using Intel Advisor and plot roofline models depicting the current state of the application's performance. These scripts can be found in the `benchmarks/user/advisor` folder of the full Devito repository. They are also available as part of the Devito package.\n", + "\n", "First, we are going to need a couple of imports to allow us to work with Devito and to run command line applications from inside this jupyter notebook. These will be needed for all three scripts." ] }, @@ -36,7 +31,7 @@ "source": [ "### Setting up the Advisor environment\n", "\n", - "Before running the following pieces of code, we must make sure that the Advisor environment, alongside the Intel C compiler `icc`, are correctly activated on the machine you wish to use the scripts on. To do so, run the following commands:\n", + "Before running the following pieces of code, we must make sure that the Advisor environment, alongside the Intel CXX compiler `icx`, are correctly activated on the machine you wish to use the scripts on. To do so, run the following commands:\n", "\n", "for Intel oneAPI:\n", "\n", @@ -48,19 +43,7 @@ "source /opt/intel/oneapi/compiler/latest/env/vars.sh \n", "```\n", "\n", - "or\n", - "\n", - "for Intel Parallel Studio:\n", - "\n", - "```shell\n", - "source /opt/intel/advisor/advixe-vars.sh\n", - "```\n", - "\n", - "```shell\n", - "source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh \n", - "```\n", - "\n", - "If your Advisor or icc have not been installed in the `/opt/intel/oneapi/advisor` (equivalently `/opt/intel/advisor`) or `/opt/intel/oneapi/compiler` (or `/opt/intel/compilers_and_libraries`) directory, replace them with your chosen path." + "If your Advisor (advixe-cl) or icx have not been installed in the `/opt/intel/oneapi/advisor` (equivalently `/opt/intel/advisor`) or `/opt/intel/oneapi/compiler` (or `/opt/intel/compilers_and_libraries`) directory, replace them with your chosen path." ] }, { @@ -98,21 +81,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;37;32mSet up multi-threading environment ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", + "\u001b[1;37;32mFound icx version: Intel(R) oneAPI DPC++/C++ Compiler 2025.0.4 (2025.0.4.20241205)\n", + "Target: x86_64-unknown-linux-gnu\n", + "Thread model: posix\n", + "InstalledDir: /opt/intel/oneapi/compiler/2025.0/bin/compiler\n", + "Configuration file: /opt/intel/oneapi/compiler/2025.0/bin/compiler/../icx.cfg\u001b[0m\n", + "\u001b[1;37;32mSetting up multi-threading environment with OpenMP ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mStarting Intel Advisor's `roofline` analysis for `JupyterProfiling`\u001b[0m\n", + "\u001b[1;37;32mProject folder: /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n", + "\u001b[1;37;32mLogging progress in: ``\u001b[0m\n", "\u001b[1;37;32mPerforming `cache warm-up` run ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mPerforming `survey` analysis ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mPerforming `tripcounts` analysis ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", - "\u001b[1;37;32mStoring `survey` and `tripcounts` data in `/home/devitouser/devito/examples/performance/profilings/JupyterProfiling`\u001b[0m\n", + "\u001b[1;37;32mStoring `survey` and `tripcounts` data in `/home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling`\u001b[0m\n", "\u001b[1;37;32mTo plot a roofline type: \u001b[0m\n", - "\u001b[1;37;32mpython3 roofline.py --name JupyterProfiling --project /home/devitouser/devito/examples/performance/profilings/JupyterProfiling --scale 1.000000\u001b[0m\n" + "\u001b[1;37;32mpython3 roofline.py --name JupyterProfiling --project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling --scale 1.000000\u001b[0m\n", + "\u001b[1;37;32m\n", + "To open the roofline using advixe-gui: \u001b[0m\n", + "\u001b[1;37;32madvixe-gui /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n" ] } ], "source": [ "#NBVAL_SKIP\n", "\n", - "! python3 $DEVITO_JUPYTER/benchmarks/user/advisor/run_advisor.py --path $DEVITO_JUPYTER/benchmarks/user/benchmark.py --exec-args \"run -P acoustic -d 128 128 128 -so 4 --tn 50 --autotune off\" --output $DEVITO_JUPYTER/examples/performance/profilings --name JupyterProfiling\n" + "! python3 $DEVITO_JUPYTER/benchmarks/user/advisor/run_advisor.py --path $DEVITO_JUPYTER/benchmarks/user/benchmark.py --exec-args \"run -P acoustic -d 64 64 64 -so 4 --tn 50 --autotune off\" --output $DEVITO_JUPYTER/examples/performance/profilings --name JupyterProfiling\n" ] }, { @@ -158,9 +151,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;37;32mOpening project...\u001b[0m\n", + "\u001b[1;37;32mOpening project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling...\u001b[0m\n", "\u001b[1;37;32mLoading data...\u001b[0m\n", - "\u001b[1;37;32mFigure saved as /home/devitouser/devito/examples/performance/resources/OverviewRoof.png.\u001b[0m\n" + "\u001b[1;37;32m\n", + "Figure saved in /home/gb4018/workspace/devitocodes/devito/benchmarks/user/advisor//home/gb4018/workspace/devitocodes/devito/examples/performance/resources/OverviewRoof.pdf.\u001b[0m\n", + "\u001b[1;37;32m\n", + "JSON file saved as /home/gb4018/workspace/devitocodes/devito/examples/performance/resources/OverviewRoof.json.\u001b[0m\n", + "\u001b[1;37;32mDone!\u001b[0m\n" ] } ], @@ -218,9 +215,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;37;32mOpening project...\u001b[0m\n", + "\u001b[1;37;32mOpening project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling...\u001b[0m\n", "\u001b[1;37;32mLoading data...\u001b[0m\n", - "\u001b[1;37;32mFigure saved as /home/devitouser/devito/examples/performance/resources/TopLoopsRoof.png.\u001b[0m\n" + "\u001b[1;37;32m\n", + "Figure saved in /home/gb4018/workspace/devitocodes/devito/benchmarks/user/advisor//home/gb4018/workspace/devitocodes/devito/examples/performance/resources/TopLoopsRoof.pdf.\u001b[0m\n", + "\u001b[1;37;32m\n", + "JSON file saved as /home/gb4018/workspace/devitocodes/devito/examples/performance/resources/TopLoopsRoof.json.\u001b[0m\n", + "\u001b[1;37;32mDone!\u001b[0m\n" ] } ], @@ -267,86 +268,6 @@ "As you can see from this roofline graph, the main point is different from the single point of the previous graph. Moreover, each point is labelled with 'Time' and 'Incidence' indicators. These represent the total execution time of each loop's main body and their percentage incidence on the total execution time of the main time loop." ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Exporting Advisor roofline data as JSON\n", - "\n", - "For easy and flexible exporting, the `advisor_to_json.py` script allows you to pack all the information represented within the previously seen graphs inside a JSON file. The command line which is needed to export our project's data in JSON format is:\n", - "```shell\n", - "python3 /benchmarks/user/advisor/advisor_to_json.py --name /examples/performance/resources/RoofsData --project /examples/performance/profilings/JupyterProfiling\n", - "```\n", - "\n", - "* `--name` specifies the name of the JSON file which will be generated\n", - "* `--project` specifies the Advisor folder which contains all the collected data about the application" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, let's run the command through bash." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1;37;32mOpening project...\u001b[0m\n", - "\u001b[1;37;32mLoading data...\u001b[0m\n", - "\u001b[1;37;32mFigure saved as /home/devitouser/devito/examples/performance/resources/RoofsData.json.\u001b[0m\n" - ] - } - ], - "source": [ - "#NBVAL_SKIP\n", - "\n", - "! python3 $DEVITO_JUPYTER/benchmarks/user/advisor/advisor_to_json.py --name $DEVITO_JUPYTER/examples/performance/resources/RoofsData --project $DEVITO_JUPYTER/examples/performance/profilings/JupyterProfiling\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now open the generated JSON file to inspect that we indeed have the data extracted from our roofline models." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'{\"roofs\": {\"memory\": [[[0, 10.044770377496254], [0, 565.556228747]], [[0, 0.3251398653506436], [0, 565.556228747]], [[0, 1.385194513137451], [0, 565.556228747]], [[0, 1.2288691632912316], [0, 565.556228747]], [[0, 3.9765711623042126], [0, 565.556228747]], [[0, 2.725070312805007], [0, 565.556228747]], [[0, 4.198971247983363], [0, 565.556228747]]], \"compute\": [[[0.3251398653506436, 32], [565.556228747, 565.556228747]], [[0.0817139523921172, 32], [142.135246015, 142.135246015]]]}, \"overview\": {\"total_ai\": 0.46157430900083685, \"total_gflops\": 46.574790645791346}}'" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_file = open(os.path.join(devito_path, 'examples/performance/resources/RoofsData.json'), 'r')\n", - "json_file.read()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's take a look at the generated JSON. Inside we can see that the file is subdivided into three sections: one to hold the points used to draw the rooflines (`roofs`), one to contain the single overview point (`overview`) and another one that contains information about the top loops as specified earlier in this tutorial (`top-loops`).\n", - "\n", - "By default, `advisor_to_json.py` collects both overview and top-loops data, but the information to collect can be specified using a `--mode` flag analogous to the one used in `roofline.py`." - ] - }, { "cell_type": "markdown", "metadata": { @@ -364,18 +285,13 @@ "* `--precision` specifies the arithmetic precision of the integral operators\n", "* `--th` specifies the threshold percentage over which to display loops in top-loops mode\n", "\n", - "`advisor_to_json.py`:\n", - "* `--scale`, as previously\n", - "* `--precision`, as previously\n", - "* `--th`, as previously\n", - "\n", "If you want to learn more about what they do, add a `--help` flag to the script that you are executing." ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -389,7 +305,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.10.12" } }, "nbformat": 4, From 5e1ba09aa801b80d114c2f3b0e90ce4fac4ed7a8 Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Tue, 11 Feb 2025 17:31:30 +0200 Subject: [PATCH 02/10] misc: Update advisor notebook --- benchmarks/user/advisor/roofline.py | 5 +- benchmarks/user/advisor/run_advisor.py | 36 ++++++----- devito/arch/archinfo.py | 26 +++++++- devito/arch/compiler.py | 12 ++-- devito/operator/profiling.py | 64 +------------------ .../performance/02_advisor_roofline.ipynb | 10 ++- tests/test_benchmark.py | 15 ++++- 7 files changed, 74 insertions(+), 94 deletions(-) diff --git a/benchmarks/user/advisor/roofline.py b/benchmarks/user/advisor/roofline.py index 852f376a54..e7a6450933 100644 --- a/benchmarks/user/advisor/roofline.py +++ b/benchmarks/user/advisor/roofline.py @@ -171,8 +171,8 @@ def roofline(name, project, scale, precision, mode, th): label_x = row.self_ai + (row.self_ai + ai_max - 2 * ai_min) * (2**0.005 - 1) label_y = row.self_gflops ax.text(label_x, label_y, - 'Time: %.2fs\n' - 'Peak pct: %.0f%%' % (row.self_time, row.percent_weight), + f'Time: {row.self_time:.2f}s\n' + f'Incidence: {row.percent_weight:.0f}%', bbox={'boxstyle': 'round', 'facecolor': 'white'}, fontsize=8) top_loops_data = [{'ai': row.self_ai, 'gflops': row.self_gflops, @@ -213,5 +213,6 @@ def roofline(name, project, scale, precision, mode, th): log('\nJSON file saved as %s.json.' % name) log('Done!') + if __name__ == '__main__': roofline() diff --git a/benchmarks/user/advisor/run_advisor.py b/benchmarks/user/advisor/run_advisor.py index e37ca38417..bbcb05f50f 100644 --- a/benchmarks/user/advisor/run_advisor.py +++ b/benchmarks/user/advisor/run_advisor.py @@ -28,37 +28,40 @@ 'in --exec-args (if any).') def run_with_advisor(path, output, name, exec_args): path = Path(path) - check(path.is_file(), '%s not found' % path) - check(path.suffix == '.py', '%s not a Python file' % path) + check(path.is_file(), f'{path} not found') + check(path.suffix == '.py', f'{path} not a Python file') # Create a directory to store the profiling report if name is None: name = path.stem if exec_args: - name = "%s_%s" % (name, ''.join(exec_args.split())) + name = f"{name}_{''.join(exec_args.split())}" if output is None: output = Path(gettempdir()).joinpath('devito-profilings') output.mkdir(parents=True, exist_ok=True) else: output = Path(output) if name is None: - output = Path(mkdtemp(dir=str(output), prefix="%s-" % name)) + output = Path(mkdtemp(dir=str(output), prefix=f"{name}-")) else: output = Path(output).joinpath(name) output.mkdir(parents=True, exist_ok=True) - # Intel Advisor and Intel compilers must be available through either - # Intel oneAPI (tested with Intel oneAPI 2025.1) + # advixe-cl and icx should be available through Intel oneAPI + # (tested with Intel oneAPI 2025.1) try: ret = check_output(['advixe-cl', '--version']).decode("utf-8") + log(f"Found advixe-cl version: {ret.strip()}\n") except FileNotFoundError: - check(False, "Error: Couldn't detect `advixe-cl` to run Intel Advisor.") + check(False, "Error: Couldn't detect `advixe-cl` to run Intel Advisor." + " Please source the Advisor environment.") try: ret = check_output(['icx', '--version']).decode("utf-8") - log("Found icx version: %s" % ret.strip()) + log(f"Found icx version: {ret.strip()}\n") except FileNotFoundError: - check(False, "Error: Couldn't detect Intel Compiler (icx).") + check(False, "Error: Couldn't detect Intel Compiler (icx)." + " Please source the Intel oneAPI compilers.") # All good, Intel compiler and advisor are available os.environ['DEVITO_ARCH'] = 'icx' @@ -82,7 +85,6 @@ def run_with_advisor(path, output, name, exec_args): ret = check_output(['numactl', '--show']).decode("utf-8") ret = dict(i.split(':') for i in ret.split('\n') if i) n_sockets = len(ret['cpubind'].split()) - n_cores = len(ret['physcpubind'].split()) # noqa except FileNotFoundError: check(False, "Couldn't detect `numactl`, necessary for thread pinning.") @@ -145,9 +147,9 @@ def run_with_advisor(path, output, name, exec_args): advixe_handler.setFormatter(advixe_formatter) advixe_logger.addHandler(advixe_handler) - log("Project folder: %s" % str(output)) - log("Logging progress in: `%s`" % str(advixe_handler)) - + log(f"Project folder: {output}") + log(f"Logging progress in: `{advixe_handler.baseFilename}`") + with progress('Performing `cache warm-up` run'): try: p_warm_up = Popen(py_cmd, stdout=PIPE, stderr=PIPE) @@ -171,13 +173,13 @@ def run_with_advisor(path, output, name, exec_args): except OSError: check(False, 'Failed!') - log('Storing `survey` and `tripcounts` data in `%s`' % str(output)) + log(f'Storing `survey` and `tripcounts` data in `{output}`') log('To plot a roofline type: ') - log('python3 roofline.py --name %s --project %s --scale %f' - % (name, str(output), n_sockets)) + log(f'python3 roofline.py --name {name} --project {output} --scale {n_sockets}') log('\nTo open the roofline using advixe-gui: ') - log('advixe-gui %s' % str(output)) + log(f'advixe-gui {output}') + if __name__ == '__main__': run_with_advisor() diff --git a/devito/arch/archinfo.py b/devito/arch/archinfo.py index 6396cc8206..02ac9aa93f 100644 --- a/devito/arch/archinfo.py +++ b/devito/arch/archinfo.py @@ -2,6 +2,7 @@ from functools import cached_property from subprocess import PIPE, Popen, DEVNULL, run +from pathlib import Path import ctypes import re import os @@ -17,8 +18,8 @@ __all__ = ['platform_registry', 'get_cpu_info', 'get_gpu_info', 'get_nvidia_cc', 'get_cuda_path', 'get_hip_path', 'check_cuda_runtime', 'get_m1_llvm_path', - 'Platform', 'Cpu64', 'Intel64', 'IntelSkylake', 'Amd', 'Arm', 'Power', - 'Device', 'NvidiaDevice', 'AmdDevice', 'IntelDevice', + 'get_advisor_path', 'Platform', 'Cpu64', 'Intel64', 'IntelSkylake', 'Amd', + 'Arm', 'Power', 'Device', 'NvidiaDevice', 'AmdDevice', 'IntelDevice', # Brand-agnostic 'ANYCPU', 'ANYGPU', # Intel CPUs @@ -464,6 +465,27 @@ def get_cuda_path(): return None +def get_advisor_path(): + """ + Detect if Intel Advisor is installed on the machine and return + its location if it is. + """ + path = None + + env_path = os.environ["PATH"] + env_path_dirs = env_path.split(":") + + for env_path_dir in env_path_dirs: + # intel/oneapi/advisor is the directory for Intel oneAPI + if "intel/advisor" in env_path_dir or "intel/oneapi/advisor" in env_path_dir: + path = Path(env_path_dir) + if path.name.startswith('bin'): + path = path.parent + return path + + return path + + @memoized_func def get_hip_path(): # *** First try: via commonly used environment variables diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index 613f11c03d..462dcd4ade 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -14,8 +14,8 @@ call_capture_output as _call_capture_output) from devito.arch import (AMDGPUX, Cpu64, AppleArm, NvidiaDevice, POWER8, POWER9, - Graviton, Cortex, IntelDevice, get_nvidia_cc, NvidiaArm, - check_cuda_runtime, get_m1_llvm_path) + Graviton, Cortex, IntelDevice, get_nvidia_cc, + check_cuda_runtime, get_m1_llvm_path, get_advisor_path) from devito.exceptions import CompilationError from devito.logger import debug, warning from devito.parameters import configuration @@ -771,17 +771,15 @@ def __init_finalize__(self, **kwargs): if configuration['profiling'] == 'advisor': # Locate the Intel Advisor installation and - # add the necessary flags to the compiler - from devito.operator.profiling import locate_intel_advisor - path = locate_intel_advisor() + # add the necessary paths and flags to the compiler + path = get_advisor_path() self.add_include_dirs(path.joinpath('include').as_posix()) _default_libs = ['ittnotify'] self.add_libraries(_default_libs) libdir = path.joinpath('lib64').as_posix() - self.add_library_dirs(libdir) - self.add_ldflags('-Wl,-rpath,%s' % libdir) + self.add_library_dirs(libdir, rpath=True) def __init_intel_mpi__(self, **kwargs): # Make sure the MPI compiler uses an Intel compiler underneath, diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index 63b671d413..27b7a9daef 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -2,10 +2,7 @@ from contextlib import contextmanager from functools import reduce from operator import mul -from pathlib import Path -from subprocess import DEVNULL, PIPE, run from time import time as seq_time -import os import cgen as c import numpy as np @@ -14,7 +11,7 @@ from devito.ir.iet import (ExpressionBundle, List, TimedList, Section, Iteration, FindNodes, Transformer) from devito.ir.support import IntervalGroup -from devito.logger import warning, error +from devito.logger import warning from devito.mpi import MPI from devito.parameters import configuration from devito.symbolics import subs_op_args @@ -367,12 +364,7 @@ class AdvisorProfiler(AdvancedProfiler): _ext_calls = [_api_resume, _api_pause] def __init__(self, name): - path = locate_intel_advisor() - if path is None: - self.initialized = False - else: - super(AdvisorProfiler, self).__init__(name) - + super(AdvisorProfiler, self).__init__(name) return def analyze(self, iet): @@ -525,55 +517,3 @@ def create_profile(name): 'advisor': AdvisorProfiler } """Profiling levels.""" - - -def locate_intel_advisor(): - """ - Detect if Intel Advisor is installed on the machine and return - its location if it is. - - """ - path = None - - try: - # Check if the directory to Intel Advisor is specified - path = Path(os.environ['DEVITO_ADVISOR_DIR']) - except KeyError: - # Otherwise, 'sniff' the location of Advisor's directory - error_msg = 'Intel Advisor cannot be found on your system, consider if you'\ - ' have sourced its environment variables correctly. Information can'\ - ' be found at https://software.intel.com/content/www/us/en/develop/'\ - 'documentation/advisor-user-guide/top/launch-the-intel-advisor/'\ - 'intel-advisor-cli/setting-and-using-intel-advisor-environment'\ - '-variables.html' - try: - res = run(["advixe-cl", "--version"], stdout=PIPE, stderr=DEVNULL) - ver = res.stdout.decode("utf-8") - if not ver: - error(error_msg) - return None - except (UnicodeDecodeError, FileNotFoundError): - error(error_msg) - return None - - env_path = os.environ["PATH"] - env_path_dirs = env_path.split(":") - - for env_path_dir in env_path_dirs: - # intel/advisor is the advisor directory for Intel Parallel Studio, - # intel/oneapi/advisor is the directory for Intel oneAPI - if "intel/advisor" in env_path_dir or "intel/oneapi/advisor" in env_path_dir: - path = Path(env_path_dir) - if path.name.startswith('bin'): - path = path.parent - - if not path: - error(error_msg) - return None - - if path.joinpath('bin64').joinpath('advixe-cl').is_file(): - return path - else: - warning("Requested `advisor` profiler, but couldn't locate executable" - "in advisor directory") - return None diff --git a/examples/performance/02_advisor_roofline.ipynb b/examples/performance/02_advisor_roofline.ipynb index e73f839a5c..5dc9eb1090 100644 --- a/examples/performance/02_advisor_roofline.ipynb +++ b/examples/performance/02_advisor_roofline.ipynb @@ -81,21 +81,25 @@ "name": "stdout", "output_type": "stream", "text": [ + "\u001b[1;37;32mFound advixe-cl version: Intel(R) Advisor 2025.0.0 (build 615865) Command Line Tool\n", + "Copyright (C) 2009-2025 Intel Corporation. All rights reserved.\n", + "\u001b[0m\n", "\u001b[1;37;32mFound icx version: Intel(R) oneAPI DPC++/C++ Compiler 2025.0.4 (2025.0.4.20241205)\n", "Target: x86_64-unknown-linux-gnu\n", "Thread model: posix\n", "InstalledDir: /opt/intel/oneapi/compiler/2025.0/bin/compiler\n", - "Configuration file: /opt/intel/oneapi/compiler/2025.0/bin/compiler/../icx.cfg\u001b[0m\n", + "Configuration file: /opt/intel/oneapi/compiler/2025.0/bin/compiler/../icx.cfg\n", + "\u001b[0m\n", "\u001b[1;37;32mSetting up multi-threading environment with OpenMP ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mStarting Intel Advisor's `roofline` analysis for `JupyterProfiling`\u001b[0m\n", "\u001b[1;37;32mProject folder: /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n", - "\u001b[1;37;32mLogging progress in: ``\u001b[0m\n", + "\u001b[1;37;32mLogging progress in: `/home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling/JupyterProfiling_2025.2.11.17.2.29.log`\u001b[0m\n", "\u001b[1;37;32mPerforming `cache warm-up` run ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mPerforming `survey` analysis ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mPerforming `tripcounts` analysis ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mStoring `survey` and `tripcounts` data in `/home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling`\u001b[0m\n", "\u001b[1;37;32mTo plot a roofline type: \u001b[0m\n", - "\u001b[1;37;32mpython3 roofline.py --name JupyterProfiling --project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling --scale 1.000000\u001b[0m\n", + "\u001b[1;37;32mpython3 roofline.py --name JupyterProfiling --project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling --scale 1\u001b[0m\n", "\u001b[1;37;32m\n", "To open the roofline using advixe-gui: \u001b[0m\n", "\u001b[1;37;32madvixe-gui /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n" diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 92ae2b36ed..b167140048 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -3,7 +3,7 @@ import sys from benchmarks.user.benchmark import run -from devito import configuration, switchconfig +from devito import configuration, switchconfig, Grid, TimeFunction, Eq, Operator from conftest import skipif from subprocess import check_call @@ -87,3 +87,16 @@ def test_run_mpi(mode): 'dump_norms': 'norms.txt' } run('acoustic', **kwargs) + + +@switchconfig(profiling='advisor') +def test_advisor_profiling(): + grid = Grid(shape=(10, 10, 10)) + + f = TimeFunction(name='f', grid=grid, space_order=2) + + eq0 = [Eq(f.forward, f.dx + 1.)] + + # Only check codegen, do not run + op = Operator(eq0) + assert 'ittnotify.h' in op._includes From 2b866d1240333eda6ba82e89715cba64af7fa62d Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Wed, 12 Feb 2025 12:02:18 +0200 Subject: [PATCH 03/10] advisor: Enhance docs --- benchmarks/user/advisor/README.md | 69 +++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/benchmarks/user/advisor/README.md b/benchmarks/user/advisor/README.md index a78f475f78..fdd8433e4a 100644 --- a/benchmarks/user/advisor/README.md +++ b/benchmarks/user/advisor/README.md @@ -1,26 +1,17 @@ -Example runs: +# Intel Advisor roofline profiling on Devito -* `python3 run_advisor.py --name isotropic --path /examples/seismic/acoustic/acoustic_example.py` -* `python3 run_advisor.py --name tti_so8 --path /examples/seismic/tti/tti_example.py --exec-args "-so 8"` -* `python3 run_advisor.py --name iso_ac_so4 --path /benchmarks/user/benchmark.py --exec-args "run -P acoustic -so 4 --tn 200 -d 100 100 100"` +This README aims to help users derive rooflines through using Devito with [Intel Advisor](https://www.intel.com/content/www/us/en/developer/tools/oneapi/advisor.html). +We recommend going through tutorial [02_advisor_roofline.ipynb](https://github.com/devitocodes/devito/blob/master/examples/performance/02_advisor_roofline.ipynb) for a more detailed step-by-step guidance. -After the run has finished you should be able to save a .json and plot the -roofline with the results: -* `python3 roofline.py --name Roofline --project ` - -To create a read-only snapshot for use with Intel Advisor GUI, use: -* `advixe-cl --snapshot --project-dir= pack -- /` - -Prerequisites: +### Prerequisites: * Support is guaranteed only for Intel oneAPI 2025; earlier versions may not work. You may download Intel oneAPI [here](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html?packages=oneapi-toolkit&oneapi-toolkit-os=linux&oneapi-lin=apt). -* Add Advisor (advixe-cl) and compilers (icx) in the path. It should be along the lines of: +* Add Advisor (advixe-cl) and compilers (icx) in the path. The right env variables should be sourced along the lines of (depending on your isntallation folder): ```sh source /opt/intel/oneapi/advisor/latest/env/vars.sh source /opt/intel/oneapi/compiler/latest/env/vars.sh ``` -depending on where you installed oneAPI * In Linux systems you may need to enable system-wide profiling by setting: @@ -38,9 +29,31 @@ sudo apt-get install numactl pip install pandas matplotlib ``` -Limitations: -* Untested with more complicated examples. +### Example runs: + +```bash +# The isotropic acoustic example +python3 run_advisor.py --name isotropic --path /examples/seismic/acoustic/acoustic_example.py +# The isotropic elastic example +python3 run_advisor.py --name iso_elastic --path /examples/seismic/elastic/elastic_example.py --exec-args "-so 4" +# The anisotropic acoustic (TTI) example +python3 run_advisor.py --name tti_so8 --path /examples/seismic/tti/tti_example.py --exec-args "-so 8" +``` + +After the run has finished you should be able to save a `.json` and plot the +roofline with the results: +```bash +python3 roofline.py --name Roofline --project +``` + +To create a read-only snapshot for use with Intel Advisor GUI, use: +```bash +advixe-cl --snapshot --project-dir= pack -- / +``` +### Limitations: + +* Not tested with all possible examples that Devito can support. * Running the `tripcounts` analysis is time-consuming, despite starting in paused mode. This analysis, together with the `survey` analysis, is necessary to generate a roofline. Both are run by `run_advisor.py`. @@ -48,16 +61,28 @@ Limitations: * Currently requires download of repository and running `pip install .`, the scripts are currently not included as a package with the user installation of Devito -TODO: +### TODO: * Give a name to the points in the roofline, otherwise it's challenging to relate loops (code sections) to data. * Emit a report summarizing the configuration used to run the analysis (threading, socket binding, ...). -Useful links: +### Useful links: + +* [ Intel® Advisor Performance Optimization Cookbook ](https://www.intel.com/content/www/us/en/docs/advisor/cookbook/2024-2/overview.html " Intel® Advisor Performance Optimization Cookbook ") + +* [ Intel® Advisor User Guide ](https://www.intel.com/content/www/us/en/docs/advisor/cookbook/2024-2/overview.html " Intel® Advisor User Guide ") + +* [ Roofline Resources for Intel® Advisor Users ](https://software.intel.com/content/www/us/en/develop/articles/advisor-roofline-resources.html " Roofline Resources for Intel® Advisor Users ") + * [ Memory-Level Roofline Analysis in Intel® Advisor ](https://software.intel.com/content/www/us/en/develop/articles/memory-level-roofline-model-with-advisor.html " Memory-Level Roofline Analysis in Intel® Advisor ") -* [CPU / Memory Roofline Insights -Perspective](https://software.intel.com/content/www/us/en/develop/documentation/advisor-user-guide/top/optimize-cpu-usage/cpu-roofline-perspective.html "CPU / Memory Roofline Insights -Perspective") -* [ Roofline Resources for Intel® Advisor Users ](https://software.intel.com/content/www/us/en/develop/articles/advisor-roofline-resources.html " Roofline Resources for Intel® Advisor Users ") \ No newline at end of file + +* [ Identify Bottlenecks Iteratively: Cache-Aware Roofline ](https://www.intel.com/content/www/us/en/docs/advisor/cookbook/2024-2/identify-bottlenecks-cache-aware-roofline.html " Identify Bottlenecks Iteratively: Cache-Aware Roofline ") + +* [ Samuel Williams, Andrew Waterman, and David Patterson [2009]. Roofline: an insightful visual performance model for multicore architectures ](https://dl.acm.org/doi/10.1145/1498765.1498785 " Roofline: an insightful visual performance model for multicore architectures ") + +* [ A. Ilic, F. Pratas and L. Sousa [2014]. Cache-aware Roofline model: Upgrading the loft ](https://ieeexplore.ieee.org/document/6506838 " Cache-aware Roofline model: Upgrading the loft ") + +* [ Understanding the Roofline Model by Durganshu Mishra ](https://hackernoon.com/understanding-the-roofline-model " Understanding the Roofline Model ") + From ca3720e600b0757872a85ced53dff814493526c5 Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Fri, 14 Feb 2025 09:00:08 +0200 Subject: [PATCH 04/10] advisor: Isolate profiler and compiler --- devito/arch/compiler.py | 12 ------------ devito/operator/operator.py | 6 ++++++ devito/operator/profiling.py | 10 +++++++++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index 462dcd4ade..67a3be9242 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -769,18 +769,6 @@ def __init_finalize__(self, **kwargs): self.__init_intel_mpi__() self.__init_intel_mpi_flags__() - if configuration['profiling'] == 'advisor': - # Locate the Intel Advisor installation and - # add the necessary paths and flags to the compiler - path = get_advisor_path() - self.add_include_dirs(path.joinpath('include').as_posix()) - - _default_libs = ['ittnotify'] - self.add_libraries(_default_libs) - - libdir = path.joinpath('lib64').as_posix() - self.add_library_dirs(libdir, rpath=True) - def __init_intel_mpi__(self, **kwargs): # Make sure the MPI compiler uses an Intel compiler underneath, # whatever the MPI distro is diff --git a/devito/operator/operator.py b/devito/operator/operator.py index 69a089e12b..5b9e0f97a1 100644 --- a/devito/operator/operator.py +++ b/devito/operator/operator.py @@ -219,6 +219,12 @@ def _build(cls, expressions, **kwargs): # Required for the jit-compilation op._compiler = kwargs['compiler'] + + # Add any metadata from the profiler + op._compiler.add_include_dirs(profiler._include_dirs) + op._compiler.add_library_dirs(profiler._lib_dirs, rpath=True) + op._compiler.add_libraries(profiler._default_libs) + op._language = kwargs['language'] op._lib = None op._cfunction = None diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index 27b7a9daef..be3509e7aa 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -8,6 +8,7 @@ import numpy as np from sympy import S +from devito.arch import get_advisor_path from devito.ir.iet import (ExpressionBundle, List, TimedList, Section, Iteration, FindNodes, Transformer) from devito.ir.support import IntervalGroup @@ -32,6 +33,9 @@ class Profiler: _default_libs = [] _ext_calls = [] + _include_dirs = [] + _lib_dirs = [] + _supports_async_sections = False _verbosity = 0 @@ -365,7 +369,11 @@ class AdvisorProfiler(AdvancedProfiler): def __init__(self, name): super(AdvisorProfiler, self).__init__(name) - return + path = get_advisor_path() + self._include_dirs = [path.joinpath('include').as_posix()] + + libdir = path.joinpath('lib64').as_posix() + self._lib_dirs = [libdir] def analyze(self, iet): return From 780d7caf1e1979746e6697b5f8a1608cc352ea1c Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Fri, 14 Feb 2025 09:44:27 +0200 Subject: [PATCH 05/10] tests: Add skipif to advisor-only test --- conftest.py | 12 +++++++++--- devito/arch/compiler.py | 2 +- tests/test_benchmark.py | 8 ++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/conftest.py b/conftest.py index 4bb0629327..df63b9488f 100644 --- a/conftest.py +++ b/conftest.py @@ -8,7 +8,7 @@ from devito import Eq, configuration, Revolver # noqa from devito.checkpointing import NoopRevolver from devito.finite_differences.differentiable import EvalDerivative -from devito.arch import Cpu64, Device, sniff_mpi_distro, Arm +from devito.arch import Cpu64, Device, sniff_mpi_distro, Arm, get_advisor_path from devito.arch.compiler import (compiler_registry, IntelCompiler, OneapiCompiler, NvidiaCompiler) from devito.ir.iet import (FindNodes, FindSymbols, Iteration, ParallelBlock, @@ -32,8 +32,8 @@ def skipif(items, whole_module=False): # Sanity check accepted = set() accepted.update({'device', 'device-C', 'device-openmp', 'device-openacc', - 'device-aomp', 'cpu64-icc', 'cpu64-icx', 'cpu64-nvc', 'cpu64-arm', - 'cpu64-icpx', 'chkpnt'}) + 'device-aomp', 'cpu64-icc', 'cpu64-icx', 'cpu64-nvc', + 'only-advisor', 'cpu64-arm', 'cpu64-icpx', 'chkpnt'}) accepted.update({'nodevice'}) unknown = sorted(set(items) - accepted) if unknown: @@ -79,6 +79,12 @@ def skipif(items, whole_module=False): isinstance(configuration['platform'], Cpu64): skipit = "`icx+cpu64` won't work with this test" break + # Skip if icx or advisor are not available + if i not in 'only-advisor' or \ + not isinstance(configuration['compiler'], IntelCompiler) or \ + not get_advisor_path(): + skipit = "Only `icx+advisor` should be tested here" + break # Skip if it won't run on Arm if i == 'cpu64-arm' and isinstance(configuration['platform'], Arm): skipit = "Arm doesn't support x86-specific instructions" diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index 67a3be9242..45b75e7c36 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -15,7 +15,7 @@ from devito.arch import (AMDGPUX, Cpu64, AppleArm, NvidiaDevice, POWER8, POWER9, Graviton, Cortex, IntelDevice, get_nvidia_cc, - check_cuda_runtime, get_m1_llvm_path, get_advisor_path) + check_cuda_runtime, get_m1_llvm_path) from devito.exceptions import CompilationError from devito.logger import debug, warning from devito.parameters import configuration diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index b167140048..87b75b299d 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -89,14 +89,18 @@ def test_run_mpi(mode): run('acoustic', **kwargs) +@skipif('only-advisor') @switchconfig(profiling='advisor') def test_advisor_profiling(): + """ + Test includes and compilation with `advisor` + """ grid = Grid(shape=(10, 10, 10)) - f = TimeFunction(name='f', grid=grid, space_order=2) eq0 = [Eq(f.forward, f.dx + 1.)] - # Only check codegen, do not run op = Operator(eq0) assert 'ittnotify.h' in op._includes + op._jit_compile() + op.apply(time_M=5) From 112d7d75afb65954355b1270ba199243621d30ed Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Fri, 14 Feb 2025 11:18:53 +0200 Subject: [PATCH 06/10] advisor: Several review edits and final touches --- conftest.py | 4 ++-- devito/arch/archinfo.py | 4 ++-- devito/operator/operator.py | 2 +- devito/operator/profiling.py | 17 ++++++++++------- tests/test_benchmark.py | 3 +-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/conftest.py b/conftest.py index df63b9488f..93de9c188c 100644 --- a/conftest.py +++ b/conftest.py @@ -33,7 +33,7 @@ def skipif(items, whole_module=False): accepted = set() accepted.update({'device', 'device-C', 'device-openmp', 'device-openacc', 'device-aomp', 'cpu64-icc', 'cpu64-icx', 'cpu64-nvc', - 'only-advisor', 'cpu64-arm', 'cpu64-icpx', 'chkpnt'}) + 'noadvisor', 'cpu64-arm', 'cpu64-icpx', 'chkpnt'}) accepted.update({'nodevice'}) unknown = sorted(set(items) - accepted) if unknown: @@ -80,7 +80,7 @@ def skipif(items, whole_module=False): skipit = "`icx+cpu64` won't work with this test" break # Skip if icx or advisor are not available - if i not in 'only-advisor' or \ + if i not in ('noadvisor',) or \ not isinstance(configuration['compiler'], IntelCompiler) or \ not get_advisor_path(): skipit = "Only `icx+advisor` should be tested here" diff --git a/devito/arch/archinfo.py b/devito/arch/archinfo.py index 02ac9aa93f..6987ce3cb6 100644 --- a/devito/arch/archinfo.py +++ b/devito/arch/archinfo.py @@ -465,6 +465,7 @@ def get_cuda_path(): return None +@memoized_func def get_advisor_path(): """ Detect if Intel Advisor is installed on the machine and return @@ -480,8 +481,7 @@ def get_advisor_path(): if "intel/advisor" in env_path_dir or "intel/oneapi/advisor" in env_path_dir: path = Path(env_path_dir) if path.name.startswith('bin'): - path = path.parent - return path + return path.parent return path diff --git a/devito/operator/operator.py b/devito/operator/operator.py index 5b9e0f97a1..cc6ae5e78a 100644 --- a/devito/operator/operator.py +++ b/devito/operator/operator.py @@ -220,7 +220,7 @@ def _build(cls, expressions, **kwargs): # Required for the jit-compilation op._compiler = kwargs['compiler'] - # Add any metadata from the profiler + # Required for compilation by the profiler op._compiler.add_include_dirs(profiler._include_dirs) op._compiler.add_library_dirs(profiler._lib_dirs, rpath=True) op._compiler.add_libraries(profiler._default_libs) diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index be3509e7aa..19871c828c 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -368,12 +368,14 @@ class AdvisorProfiler(AdvancedProfiler): _ext_calls = [_api_resume, _api_pause] def __init__(self, name): - super(AdvisorProfiler, self).__init__(name) + super().__init__(name) path = get_advisor_path() - self._include_dirs = [path.joinpath('include').as_posix()] - - libdir = path.joinpath('lib64').as_posix() - self._lib_dirs = [libdir] + if path: + self._include_dirs = [path.joinpath('include').as_posix()] + libdir = path.joinpath('lib64').as_posix() + self._lib_dirs = [libdir] + else: + self.initialized = False def analyze(self, iet): return @@ -508,8 +510,9 @@ def create_profile(name): if profiler.initialized: return profiler else: - warning("Couldn't set up `%s` profiler; reverting to `advanced`" % level) - profiler = profiler_registry['basic'](name) + rlevel = 'advanced' + warning(f"Couldn't set up `{level}` profiler; reverting to `{rlevel}`") + profiler = profiler_registry[rlevel](name) # We expect the `advanced` profiler to always initialize successfully assert profiler.initialized return profiler diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 87b75b299d..513776489c 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -89,7 +89,7 @@ def test_run_mpi(mode): run('acoustic', **kwargs) -@skipif('only-advisor') +@skipif('noadvisor') @switchconfig(profiling='advisor') def test_advisor_profiling(): """ @@ -102,5 +102,4 @@ def test_advisor_profiling(): op = Operator(eq0) assert 'ittnotify.h' in op._includes - op._jit_compile() op.apply(time_M=5) From a53b59af7418721e6b138cc85082b88cbf901e51 Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Fri, 14 Feb 2025 21:14:46 +0200 Subject: [PATCH 07/10] advisor: More edits --- devito/operator/profiling.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index 19871c828c..c9b0bad3a3 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -55,6 +55,12 @@ def __init__(self, name): self.initialized = True + def add_include_dir(self, dir_path): + self._include_dirs.append(dir_path) + + def add_lib_dir(self, dir_path): + self._lib_dirs.append(dir_path) + def analyze(self, iet): """ Analyze the Sections in the given IET. This populates `self._sections`. @@ -371,9 +377,8 @@ def __init__(self, name): super().__init__(name) path = get_advisor_path() if path: - self._include_dirs = [path.joinpath('include').as_posix()] - libdir = path.joinpath('lib64').as_posix() - self._lib_dirs = [libdir] + self.add_include_dir(path.joinpath('include').as_posix()) + self.add_lib_dir(path.joinpath('lib64').as_posix()) else: self.initialized = False @@ -510,9 +515,8 @@ def create_profile(name): if profiler.initialized: return profiler else: - rlevel = 'advanced' - warning(f"Couldn't set up `{level}` profiler; reverting to `{rlevel}`") - profiler = profiler_registry[rlevel](name) + warning(f"Couldn't set up `{level}` profiler; reverting to 'advanced'") + profiler = profiler_registry['advanced'](name) # We expect the `advanced` profiler to always initialize successfully assert profiler.initialized return profiler From ecb58fbb48939a3508360fd0eb90affaf01db19d Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Mon, 17 Feb 2025 13:50:18 +0200 Subject: [PATCH 08/10] advisor: Turn class attrs to instance attrs --- devito/arch/compiler.py | 2 +- devito/operator/profiling.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index 45b75e7c36..aa8cfa0795 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -14,7 +14,7 @@ call_capture_output as _call_capture_output) from devito.arch import (AMDGPUX, Cpu64, AppleArm, NvidiaDevice, POWER8, POWER9, - Graviton, Cortex, IntelDevice, get_nvidia_cc, + Graviton, Cortex, IntelDevice, get_nvidia_cc, NvidiaArm, check_cuda_runtime, get_m1_llvm_path) from devito.exceptions import CompilationError from devito.logger import debug, warning diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index c9b0bad3a3..b314a5d69a 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -33,9 +33,6 @@ class Profiler: _default_libs = [] _ext_calls = [] - _include_dirs = [] - _lib_dirs = [] - _supports_async_sections = False _verbosity = 0 @@ -53,6 +50,10 @@ def __init__(self, name): # Python-level timers self.py_timers = OrderedDict() + # Instance attributes + self._include_dirs = [] + self._lib_dirs = [] + self.initialized = True def add_include_dir(self, dir_path): @@ -375,6 +376,7 @@ class AdvisorProfiler(AdvancedProfiler): def __init__(self, name): super().__init__(name) + path = get_advisor_path() if path: self.add_include_dir(path.joinpath('include').as_posix()) @@ -383,6 +385,9 @@ def __init__(self, name): self.initialized = False def analyze(self, iet): + """ + A no-op, as the Advisor profiler does not need to analyze the IET. + """ return def instrument(self, iet, timer): From c77c4d39723aa7e74750a35dab6909568179c96e Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Wed, 26 Feb 2025 20:38:15 +0200 Subject: [PATCH 09/10] misc: Avoid reinitialization of parent class and minor fixes --- benchmarks/user/advisor/run_advisor.py | 2 +- conftest.py | 6 ++--- devito/operator/profiling.py | 34 ++++++++++++-------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/benchmarks/user/advisor/run_advisor.py b/benchmarks/user/advisor/run_advisor.py index bbcb05f50f..9c0d1554ff 100644 --- a/benchmarks/user/advisor/run_advisor.py +++ b/benchmarks/user/advisor/run_advisor.py @@ -8,7 +8,7 @@ from subprocess import check_output, PIPE, Popen from tempfile import gettempdir, mkdtemp -from advisor_logging import (check, log, progress, log_process) +from advisor_logging import check, log, progress, log_process @click.command() diff --git a/conftest.py b/conftest.py index 93de9c188c..5a1bc481b9 100644 --- a/conftest.py +++ b/conftest.py @@ -80,9 +80,9 @@ def skipif(items, whole_module=False): skipit = "`icx+cpu64` won't work with this test" break # Skip if icx or advisor are not available - if i not in ('noadvisor',) or \ - not isinstance(configuration['compiler'], IntelCompiler) or \ - not get_advisor_path(): + if i == 'noadvisor' and \ + (not isinstance(configuration['compiler'], IntelCompiler) or + not get_advisor_path()): skipit = "Only `icx+advisor` should be tested here" break # Skip if it won't run on Arm diff --git a/devito/operator/profiling.py b/devito/operator/profiling.py index b314a5d69a..30dd6a2cfb 100644 --- a/devito/operator/profiling.py +++ b/devito/operator/profiling.py @@ -33,10 +33,15 @@ class Profiler: _default_libs = [] _ext_calls = [] + _include_dirs = [] + _lib_dirs = [] + _supports_async_sections = False _verbosity = 0 + _attempted_init = False + def __init__(self, name): self.name = name @@ -50,17 +55,7 @@ def __init__(self, name): # Python-level timers self.py_timers = OrderedDict() - # Instance attributes - self._include_dirs = [] - self._lib_dirs = [] - - self.initialized = True - - def add_include_dir(self, dir_path): - self._include_dirs.append(dir_path) - - def add_lib_dir(self, dir_path): - self._lib_dirs.append(dir_path) + self._attempted_init = True def analyze(self, iet): """ @@ -362,9 +357,6 @@ class AdvisorProfiler(AdvancedProfiler): """ Rely on Intel Advisor 2025.1 for performance profiling. - Tested versions of Intel Advisor: - - As contained in Intel Parallel Studio 2020 v 2020 Update 2 - - As contained in Intel oneAPI 2021 beta08 """ _api_resume = '__itt_resume' @@ -375,14 +367,18 @@ class AdvisorProfiler(AdvancedProfiler): _ext_calls = [_api_resume, _api_pause] def __init__(self, name): + if self._attempted_init: + return + super().__init__(name) path = get_advisor_path() if path: - self.add_include_dir(path.joinpath('include').as_posix()) - self.add_lib_dir(path.joinpath('lib64').as_posix()) + self._include_dirs.append(path.joinpath('include').as_posix()) + self._lib_dirs.append(path.joinpath('lib64').as_posix()) + self._attempted_init = True else: - self.initialized = False + self._attempted_init = False def analyze(self, iet): """ @@ -517,13 +513,13 @@ def create_profile(name): level = configuration['profiling'] profiler = profiler_registry[level](name) - if profiler.initialized: + if profiler._attempted_init: return profiler else: warning(f"Couldn't set up `{level}` profiler; reverting to 'advanced'") profiler = profiler_registry['advanced'](name) # We expect the `advanced` profiler to always initialize successfully - assert profiler.initialized + assert profiler._attempted_init return profiler From 3c514da89bf225adee8adc2988318a8c20456465 Mon Sep 17 00:00:00 2001 From: George Bisbas Date: Tue, 4 Mar 2025 13:44:10 +0200 Subject: [PATCH 10/10] examples: Drop dev env specific logs --- .../performance/02_advisor_roofline.ipynb | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/performance/02_advisor_roofline.ipynb b/examples/performance/02_advisor_roofline.ipynb index 5dc9eb1090..dc76d7ecf0 100644 --- a/examples/performance/02_advisor_roofline.ipynb +++ b/examples/performance/02_advisor_roofline.ipynb @@ -92,17 +92,17 @@ "\u001b[0m\n", "\u001b[1;37;32mSetting up multi-threading environment with OpenMP ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mStarting Intel Advisor's `roofline` analysis for `JupyterProfiling`\u001b[0m\n", - "\u001b[1;37;32mProject folder: /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n", - "\u001b[1;37;32mLogging progress in: `/home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling/JupyterProfiling_2025.2.11.17.2.29.log`\u001b[0m\n", + "\u001b[1;37;32mProject folder: /home/devitouser/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n", + "\u001b[1;37;32mLogging progress in: `/home/devitouser/devito/examples/performance/profilings/JupyterProfiling/JupyterProfiling_2025.2.11.17.2.29.log`\u001b[0m\n", "\u001b[1;37;32mPerforming `cache warm-up` run ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mPerforming `survey` analysis ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", "\u001b[1;37;32mPerforming `tripcounts` analysis ... \u001b[0m\u001b[1;37;32mDone!\u001b[0m\n", - "\u001b[1;37;32mStoring `survey` and `tripcounts` data in `/home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling`\u001b[0m\n", + "\u001b[1;37;32mStoring `survey` and `tripcounts` data in `/home/devitouser/devito/examples/performance/profilings/JupyterProfiling`\u001b[0m\n", "\u001b[1;37;32mTo plot a roofline type: \u001b[0m\n", - "\u001b[1;37;32mpython3 roofline.py --name JupyterProfiling --project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling --scale 1\u001b[0m\n", + "\u001b[1;37;32mpython3 roofline.py --name JupyterProfiling --project /home/devitouser/devito/examples/performance/profilings/JupyterProfiling --scale 1\u001b[0m\n", "\u001b[1;37;32m\n", "To open the roofline using advixe-gui: \u001b[0m\n", - "\u001b[1;37;32madvixe-gui /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n" + "\u001b[1;37;32madvixe-gui /home/devitouser/devito/examples/performance/profilings/JupyterProfiling\u001b[0m\n" ] } ], @@ -155,12 +155,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;37;32mOpening project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling...\u001b[0m\n", + "\u001b[1;37;32mOpening project /home/devitouser/devito/examples/performance/profilings/JupyterProfiling...\u001b[0m\n", "\u001b[1;37;32mLoading data...\u001b[0m\n", "\u001b[1;37;32m\n", - "Figure saved in /home/gb4018/workspace/devitocodes/devito/benchmarks/user/advisor//home/gb4018/workspace/devitocodes/devito/examples/performance/resources/OverviewRoof.pdf.\u001b[0m\n", + "Figure saved in /home/devitouser/devito/benchmarks/user/advisor//home/devitouser/devito/examples/performance/resources/OverviewRoof.pdf.\u001b[0m\n", "\u001b[1;37;32m\n", - "JSON file saved as /home/gb4018/workspace/devitocodes/devito/examples/performance/resources/OverviewRoof.json.\u001b[0m\n", + "JSON file saved as /home/devitouser/devito/examples/performance/resources/OverviewRoof.json.\u001b[0m\n", "\u001b[1;37;32mDone!\u001b[0m\n" ] } @@ -219,12 +219,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;37;32mOpening project /home/gb4018/workspace/devitocodes/devito/examples/performance/profilings/JupyterProfiling...\u001b[0m\n", + "\u001b[1;37;32mOpening project /home/devitouser/devito/examples/performance/profilings/JupyterProfiling...\u001b[0m\n", "\u001b[1;37;32mLoading data...\u001b[0m\n", "\u001b[1;37;32m\n", - "Figure saved in /home/gb4018/workspace/devitocodes/devito/benchmarks/user/advisor//home/gb4018/workspace/devitocodes/devito/examples/performance/resources/TopLoopsRoof.pdf.\u001b[0m\n", + "Figure saved in /home/devitouser/devito/benchmarks/user/advisor//home/devitouser/devito/examples/performance/resources/TopLoopsRoof.pdf.\u001b[0m\n", "\u001b[1;37;32m\n", - "JSON file saved as /home/gb4018/workspace/devitocodes/devito/examples/performance/resources/TopLoopsRoof.json.\u001b[0m\n", + "JSON file saved as /home/devitouser/devito/examples/performance/resources/TopLoopsRoof.json.\u001b[0m\n", "\u001b[1;37;32mDone!\u001b[0m\n" ] }