From 652975250eabd5ef1365e7e3496bf4bd418dec3d Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Mon, 27 Jul 2020 10:27:15 -0400 Subject: [PATCH] Testing: tc4 installs python-netCDF4 via venv (#1166) Squash merge of following commits: * Testing: tc4 installs python-netCDF4 via venv Currently, users are expected to have numpy and netcdf4 python modules in order to generate the necessary netCDF input files. This fails in environments where these modules are unavailable. This patch now installs the modules into a virtual environment which are accessible when generating the tc4 inputs. This solution is local to tc4 but could be extended to other tests as needed. * Testing: remove Python numpy and netCDF4 modules The numpy and netCDF4 packages are no longer needed since tc4 now installs these locally. * Test: Install virtualenv for Python2 tc4 was using the venv module which appears to be python3-specific, and Travis Ubuntu defaults to python2. Also virtualenv was not installed in either case. This patch adds python-virtualenv to the install packages and uses the virtualenv module. * Test: Add python-dev for ARM64 numpy Travis ARM nodes need to build numpy natively when installed by pip, and thus require Python headers. These are provided by the python-dev package. * Test: Revert to Py3 for tc4 generation scripts Reverting the python 2 support (default for Travis) to use Python 3 syntax. The main reason is that Python 3 includes venv (equivalent to virtualenv) as its standard library, and is therefore guaranteed to exist if Python 3 exists. Python 3's virtualenv must be independently installed, which cannot be confirmed. This will cause problems for people without Python 3, but this is probably the best solution, or at least the starting point for a more general solution. * Test: Adding python3-venv to Travis Ubuntu apparently requires an explicit install of python3-venv despite it being part of the standard library. Go figure... * Test: Arm64 tc4 Configuration support Arm64 Ubuntu environments require explicit installations which are otherwise provided on x86 Ubuntu: * Python 3 Pip must be installed (python3-pip) * Wheel installation must be explicitly installed * Cython is required for numpy * Numpy must be explicitly built before installing python-netCDF4 * Test: tc4 common Python local-env Build times for setting up the virtual environments can be very expensive on the Arm64 Ubuntu nodes, so we now create a shared directory for launching the environments. * Test: Use '.' in place of 'source' for Make * Test: Test for required Python modules in tc4 We have reworked the Makefile to conditionally test for required Python modules in tc4. If unavailable, we install these in a virtual environment. This does not address many scenarios, such as if Python 3 is missing, venv is missing (as in Ubuntu), or handle the situation if they do not exist. It assumes that either the modules exist, or that they can be installed by venv. This should be seen as an iterative step to get things working on Travis x86 and Arm64, as well as GFDL's Gaea and most user Linux platforms. * Test: Explicit python execs for tc4 input build This resolves some issues with python2/3 resolution and limited support of various platforms for module support. Specifically, older platform with basic Python 3 support may not also have numpy support. In this case, we can defer back to Python 2 (or whatever the system Python may be). * Test: Setup Python venv at build time This patch moves the Python virtual environment configuration to the main Makefile, which is setup at build time, rather than in the model configuration Makefile, which will typically not have internet access if run on a compute node. As before, the venv will only be setup when the numpy and netCDF4 modules are unavailable. A minor bug in the logic of the check has also been fixed. --- .testing/Makefile | 54 ++++++++++++++++++++++++++++++++++++++--- .testing/tc4/.gitignore | 4 +++ .testing/tc4/Makefile | 7 +++++- .travis.yml | 1 + 4 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 .testing/tc4/.gitignore diff --git a/.testing/Makefile b/.testing/Makefile index d1ceb16577..ab978fdadc 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -92,11 +92,34 @@ TARGET_SOURCE = $(call SOURCE,build/target_codebase/src) \ $(wildcard build/target_codebase/config_src/ext*/*.F90) FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src) +#--- +# Python preprocessing environment configuration + +HAS_NUMPY = $(shell python -c "import numpy" 2> /dev/null && echo "yes") +HAS_NETCDF4 = $(shell python -c "import netCDF4" 2> /dev/null && echo "yes") + +USE_VENV = +ifneq ($(HAS_NUMPY), yes) + USE_VENV = yes +endif +ifneq ($(HAS_NETCDF4), yes) + USE_VENV = yes +endif + +# When disabled, activation is a null operation (`true`) +VENV_PATH = +VENV_ACTIVATE = true +ifeq ($(USE_VENV), yes) + VENV_PATH = work/local-env + VENV_ACTIVATE = . $(VENV_PATH)/bin/activate +endif + + #--- # Rules .PHONY: all build.regressions -all: $(foreach b,$(BUILDS),build/$(b)/MOM6) +all: $(foreach b,$(BUILDS),build/$(b)/MOM6) $(VENV_PATH) build.regressions: $(foreach b,symmetric target,build/$(b)/MOM6) # Executable @@ -184,6 +207,18 @@ $(LIST_PATHS) $(MKMF): cd $(DEPS)/mkmf; git checkout $(MKMF_COMMIT) +#--- +# Python preprocessing +# NOTE: Some less mature environments (e.g. Arm64 Ubuntu) require explicit +# installation of numpy before netCDF4, as well as wheel and cython support. +work/local-env: + python3 -m venv $@ + . $@/bin/activate \ + && pip3 install wheel \ + && pip3 install cython \ + && pip3 install numpy \ + && pip3 install netCDF4 + #---- # Testing @@ -264,7 +299,6 @@ $(eval $(call CMP_RULE,regression,symmetric target)) # TODO: chksum_diag parsing of restart files - #--- # Test run output files @@ -281,7 +315,13 @@ work/%/$(1)/ocean.stats work/%/$(1)/chksum_diag: build/$(2)/MOM6 if [ $(3) ]; then find build/$(2) -name *.gcda -exec rm -f '{}' \; ; fi mkdir -p $$(@D) cp -rL $$*/* $$(@D) - cd $$(@D) && if [ -f Makefile ]; then $(MAKE); fi + if [ -f $$(@D)/Makefile ]; then \ + $$(VENV_ACTIVATE) \ + && cd $$(@D) \ + && $(MAKE); \ + else \ + cd $$(@D); \ + fi mkdir -p $$(@D)/RESTART echo -e "$(4)" > $$(@D)/MOM_override cd $$(@D) \ @@ -327,7 +367,13 @@ work/%/restart/ocean.stats: build/symmetric/MOM6 rm -rf $(@D) mkdir -p $(@D) cp -rL $*/* $(@D) - cd work/$*/restart && if [ -f Makefile ]; then $(MAKE); fi + if [ -f $(@D)/Makefile ]; then \ + $(VENV_ACTIVATE) \ + && cd work/$*/restart \ + && $(MAKE); \ + else \ + cd work/$*/restart; \ + fi mkdir -p $(@D)/RESTART # Generate the half-period input namelist # TODO: Assumes that runtime set by DAYMAX, will fail if set by input.nml diff --git a/.testing/tc4/.gitignore b/.testing/tc4/.gitignore new file mode 100644 index 0000000000..29f62fb208 --- /dev/null +++ b/.testing/tc4/.gitignore @@ -0,0 +1,4 @@ +ocean_hgrid.nc +sponge.nc +temp_salt_ic.nc +topog.nc diff --git a/.testing/tc4/Makefile b/.testing/tc4/Makefile index c332bbd7e6..a9aa395b9c 100644 --- a/.testing/tc4/Makefile +++ b/.testing/tc4/Makefile @@ -1,3 +1,8 @@ -ocean_hgrid.nc topog.nc temp_salt_ic.nc sponge.nc: +OUT=ocean_hgrid.nc sponge.nc temp_salt_ic.nc topog.nc + +$(OUT): python build_grid.py python build_data.py + +clean: + rm -rf $(OUT) diff --git a/.travis.yml b/.travis.yml index 6bf509ce8c..10816b7122 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ addons: - mpich libmpich-dev - doxygen graphviz flex bison cmake - python-numpy python-netcdf4 + - python3 python3-dev python3-venv python3-pip - bc jobs: