From 86e55190d1014f0dd7ee2b827ea4dce4bae17c67 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Aug 2023 11:56:51 -0600 Subject: [PATCH 01/17] Per #12, replace Centos image with Debian one --- .github/jobs/build_docker_image.sh | 4 +- Dockerfile.debian | 72 ------------------------------ Dockerfile.debian_unit_test_env | 26 ----------- 3 files changed, 2 insertions(+), 100 deletions(-) delete mode 100644 Dockerfile.debian delete mode 100644 Dockerfile.debian_unit_test_env diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh index 79214e9..dd973a5 100755 --- a/.github/jobs/build_docker_image.sh +++ b/.github/jobs/build_docker_image.sh @@ -5,7 +5,7 @@ MET_BASE_TAG=${GITHUB_TAG} DOCKERHUB_TAG_BASE=${DOCKERHUB_BASE_REPO}:${MET_BASE_TAG} -DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile.debian +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build_base_image.log @@ -18,7 +18,7 @@ fi DOCKERHUB_TAG_UNIT_TEST=${DOCKERHUB_UNIT_TEST_REPO}:${GITHUB_TAG} -DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile.debian_unit_test_env +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/Dockerfile.unit_test_env CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build_unit_test_env.log diff --git a/Dockerfile.debian b/Dockerfile.debian deleted file mode 100644 index eb5f6c0..0000000 --- a/Dockerfile.debian +++ /dev/null @@ -1,72 +0,0 @@ -ARG DEBIAN_VERSION=10 -FROM debian:${DEBIAN_VERSION}-slim -MAINTAINER George McCabe - -# -# Define the compilers. -# -ENV CC /usr/bin/gcc -ENV CXX /usr/bin/g++ -ENV FC /usr/bin/gfortran -ENV F77 /usr/bin/gfortran - -ENV PYTHON_VER 3.10.4 - -ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz - -# -# Setup the environment for interactive bash shell. -# Set soft limit to unlimited to prevent GRIB2 seg faults -# -RUN echo export MET_BASE=/usr/local/share/met >> /root/.bashrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /root/.bashrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /root/.bashrc \ - && echo ulimit -S -s unlimited >> /root/.bashrc - -ENV MET_FONT_DIR /usr/local/share/met/fonts - -# Install required system tools -RUN apt update && apt -y upgrade \ - && apt install -y build-essential gfortran wget unzip curl imagemagick \ - libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ - emacs less \ - libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev \ - libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev - -# -# Install Python -# https://linuxhint.com/install-python-debian-10/ -# -RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ - && tar xzf Python-${PYTHON_VER}.tgz \ - && cd Python-${PYTHON_VER} \ - && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ - && make -j `nproc` \ - && make install \ - && python3 -m pip install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - -RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ - && mkdir -p /usr/local/share/met \ - && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met - -# Fix rules for ghostscript files in convert -# See: https://en.linuxportal.info/tutorials/troubleshooting/how-to-fix-errors-from-imagemagick-imagick-conversion-system-security-policy -# -RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="coder" rights="read | write" pattern="PS/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="EPS"/policy domain="coder" rights="read | write" pattern="EPS"/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read | write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="XPS"/policy domain="coder" rights="read | write" pattern="XPS"/g' /etc/ImageMagick-6/policy.xml - -# -# Set the working directory. -# -WORKDIR /met - -RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ - && tar -zxf tar_files.tgz \ - && export SKIP_MET=yes \ - && chmod +x compile_MET_all.sh \ - && ./compile_MET_all.sh development.docker diff --git a/Dockerfile.debian_unit_test_env b/Dockerfile.debian_unit_test_env deleted file mode 100644 index 6938adb..0000000 --- a/Dockerfile.debian_unit_test_env +++ /dev/null @@ -1,26 +0,0 @@ -ARG MET_BASE_TAG - -FROM dtcenter/met-base:${MET_BASE_TAG} - -MAINTAINER John Halley Gotway - -# -# Set the working directory. -# -WORKDIR /met - -# -# Download and install MET and GhostScript fonts. -# Delete the MET source code for tagged releases matching "v"*. -# -RUN echo "Installing tools needed for running MET unit tests..." \ - && echo "Installing Perl XML Parser..." \ - && apt install -y libxml-parser-perl \ - && echo "Installing R..." \ - && apt install -y r-base \ - && echo "Installing R ncdf4 1.21..." \ - && wget https://cran.r-project.org/src/contrib/ncdf4_1.21.tar.gz \ - && R CMD INSTALL --configure-args="--with-nc-config=/usr/local/netcdf/netcdf-c-4.7.4/nc-config" ncdf4_1.21.tar.gz \ - && echo "Installing NCO and netcdf-bin (for ncdiff)..." \ - && apt install -y nco netcdf-bin \ - && echo "Finished installing unit test tools" From 5027bfc3c087ebfd6dbf975742a33a2be3865bbd Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Aug 2023 11:57:30 -0600 Subject: [PATCH 02/17] Per #12, incorporate Debian updates --- Dockerfile | 188 ++++++++++----------------------------- Dockerfile.unit_test_env | 15 ++-- 2 files changed, 53 insertions(+), 150 deletions(-) diff --git a/Dockerfile b/Dockerfile index 469f16d..eb5f6c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -FROM centos:7 -MAINTAINER John Halley Gotway +ARG DEBIAN_VERSION=10 +FROM debian:${DEBIAN_VERSION}-slim +MAINTAINER George McCabe # # Define the compilers. @@ -9,160 +10,63 @@ ENV CXX /usr/bin/g++ ENV FC /usr/bin/gfortran ENV F77 /usr/bin/gfortran -# -# Define library versions. -# Match WCOSS2 versions as of October, 2022. -# -ENV HDF5_VER 1_10_6 -ENV NETCDF4C_VER 4.7.4 -ENV NETCDF4CXX_VER 4.3.1 +ENV PYTHON_VER 3.10.4 -# -# Define package URL's. -# -ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz -ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z - -ENV HDF5_URL https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-${HDF5_VER}.zip -ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF4C_VER}.zip -ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v${NETCDF4CXX_VER}.tar.gz - -ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz # -# Install the required packages. -# -RUN yum -y update \ - && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ - libpng-devel jasper jasper-devel zlib zlib-devel \ - cairo-devel freetype-devel epel-release \ - hostname m4 make tar tcsh ksh time wget which \ - flex flex-devel bison bison-devel unzip \ - && yum -y install git g2clib-devel gsl-devel \ - && yum -y install sqlite-devel libffi-devel libcurl-devel.x86_64 \ - && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ - && yum -y install python3 python3-devel python3-pip - -# Needed to prevent error in subsequent pip command -ENV LD_LIBRARY_PATH /usr/local/lib - -RUN pip3 install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - -# -# Set the working directory. -# -WORKDIR /met - -# -# Setup the environment for interactive bash/csh container shells. +# Setup the environment for interactive bash shell. +# Set soft limit to unlimited to prevent GRIB2 seg faults # -RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ - && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ - && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ - && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc +RUN echo export MET_BASE=/usr/local/share/met >> /root/.bashrc \ + && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /root/.bashrc \ + && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /root/.bashrc \ + && echo ulimit -S -s unlimited >> /root/.bashrc ENV MET_FONT_DIR /usr/local/share/met/fonts -# -# Download and install BUFRLIB. -# -RUN mkdir -p /met/logs \ - && mkdir -p /met/external_libs/BUFRLIB \ - && cd /met/external_libs/BUFRLIB \ - && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ - && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ - && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ - && chmod +x preproc_patch.sh \ - && LOG_FILE=/met/logs/BUFRLIB_build.log \ - && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ - && ./preproc_patch.sh *.F > ${LOG_FILE} \ - && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ - && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ - && ar crv libbufr.a *.o >> ${LOG_FILE} \ - && rm -f /usr/local/lib/libbufr.a \ - && cp *.a /usr/local/lib \ - && cd /met/external_libs \ - && rm -rf BUFRLIB +# Install required system tools +RUN apt update && apt -y upgrade \ + && apt install -y build-essential gfortran wget unzip curl imagemagick \ + libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ + emacs less \ + libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev \ + libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev # -# Download and install HDF5. +# Install Python +# https://linuxhint.com/install-python-debian-10/ # -RUN mkdir -p /met/external_libs/hdf5 \ - && cd /met/external_libs/hdf5 \ - && echo "Downloading HDF5 from ${HDF5_URL}" \ - && wget ${HDF5_URL} \ - && unzip hdf5-${HDF5_VER}.zip \ - && cd hdf5-hdf5-${HDF5_VER} \ - && LOG_FILE=/met/logs/hdf5-hdf5-${HDF5_VER}_configure.log \ - && echo "Configuring hdf5-hdf5-${HDF5_VER} and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local --enable-cxx --with-default-api-version=v18 > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdf5-hdf5-${HDF5_VER}_make_install.log \ - && echo "Compiling hdf5-hdf5-${HDF5_VER} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf hdf5 +RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ + && tar xzf Python-${PYTHON_VER}.tgz \ + && cd Python-${PYTHON_VER} \ + && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ + && make -j `nproc` \ + && make install \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install numpy xarray netCDF4 +RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ + && mkdir -p /usr/local/share/met \ + && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met + +# Fix rules for ghostscript files in convert +# See: https://en.linuxportal.info/tutorials/troubleshooting/how-to-fix-errors-from-imagemagick-imagick-conversion-system-security-policy # -# Download and install NetCDF4 (C and C++). -# -RUN mkdir -p /met/external_libs/netcdf \ - && cd /met/external_libs/netcdf \ - && echo "Downloading netcdf-c-${NETCDF4C_VER} from ${NETCDF4C_URL}" \ - && wget ${NETCDF4C_URL} \ - && unzip v${NETCDF4C_VER}.zip \ - && cd netcdf-c-${NETCDF4C_VER} \ - && LOG_FILE=/met/logs/netcdf-c-${NETCDF4C_VER}_configure.log \ - && echo "Configuring netcdf-c-${NETCDF4C_VER} and writing log file ${LOG_FILE}" \ - && ./configure --enable-shared --enable-netcdf-4 > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-c-${NETCDF4C_VER}_make_install.log \ - && echo "Compiling netcdf-c-${NETCDF4C_VER} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading from ${NETCDF4CXX_URL}" \ - && cd /met/external_libs/netcdf \ - && wget ${NETCDF4CXX_URL} \ - && tar -xzf v${NETCDF4CXX_VER}.tar.gz \ - && cd netcdf-cxx4-${NETCDF4CXX_VER} \ - && LOG_FILE=/met/logs/netcdf-cxx4-${NETCDF4CXX_VER}_configure.log \ - && echo "Configuring netcdf-cxx4-${NETCDF4CXX_VER} and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-cxx4-${NETCDF4CXX_VER}_make_install.log \ - && echo "Compiling netcdf-cxx4-${NETCDF4CXX_VER} and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf netcdf +RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="coder" rights="read | write" pattern="PS/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="EPS"/policy domain="coder" rights="read | write" pattern="EPS"/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read | write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="XPS"/policy domain="coder" rights="read | write" pattern="XPS"/g' /etc/ImageMagick-6/policy.xml # -# Download and install HDF4 and HDFEOS. +# Set the working directory. # -RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ - && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/HDF4.2r3 \ - && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ - && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ - && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ - && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ - && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ - && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading hdfeos from ${HDFEOS_URL}" \ - && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/hdfeos \ - && LOG_FILE=/met/logs/hdfeos_configure.log \ - && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdfeos_make_install.log \ - && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && mkdir /usr/local/hdfeos/include \ - && cp include/*.h /usr/local/hdfeos/include/. \ - && cd /met/external_libs \ - && rm -rf HDF4.2r3 hdfeos +WORKDIR /met -RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ - && mkdir -p /usr/local/share/met \ - && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met +RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ + && tar -zxf tar_files.tgz \ + && export SKIP_MET=yes \ + && chmod +x compile_MET_all.sh \ + && ./compile_MET_all.sh development.docker diff --git a/Dockerfile.unit_test_env b/Dockerfile.unit_test_env index d95255e..6938adb 100644 --- a/Dockerfile.unit_test_env +++ b/Dockerfile.unit_test_env @@ -15,13 +15,12 @@ WORKDIR /met # RUN echo "Installing tools needed for running MET unit tests..." \ && echo "Installing Perl XML Parser..." \ - && yum makecache \ - && yum -y install perl-XML-Parser \ + && apt install -y libxml-parser-perl \ && echo "Installing R..." \ - && yum -y install R \ - && echo "Installing R ncdf4 1.19..." \ - && wget https://cran.r-project.org/src/contrib/ncdf4_1.19.tar.gz \ - && R CMD INSTALL ncdf4_1.19.tar.gz \ - && echo "Installing NCO (for ncdiff)..." \ - && yum -y install nco \ + && apt install -y r-base \ + && echo "Installing R ncdf4 1.21..." \ + && wget https://cran.r-project.org/src/contrib/ncdf4_1.21.tar.gz \ + && R CMD INSTALL --configure-args="--with-nc-config=/usr/local/netcdf/netcdf-c-4.7.4/nc-config" ncdf4_1.21.tar.gz \ + && echo "Installing NCO and netcdf-bin (for ncdiff)..." \ + && apt install -y nco netcdf-bin \ && echo "Finished installing unit test tools" From c954286a1837fc1bb5389e59efa8d4b6e8a68809 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Aug 2023 13:10:26 -0600 Subject: [PATCH 03/17] Per #12, update the logic for retrieving the tag name to avoid the following GHA warning message: Warning: The command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ --- .github/workflows/build_and_push_docker.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push_docker.yml b/.github/workflows/build_and_push_docker.yml index b02ca58..a582c1e 100644 --- a/.github/workflows/build_and_push_docker.yml +++ b/.github/workflows/build_and_push_docker.yml @@ -19,12 +19,15 @@ jobs: - name: Get tag name id: get_tag_name - run: echo ::set-output name=tag_name::${GITHUB_REF#"refs/tags/"} + uses: devops-actions/action-get-tag@v1.0.1 + with: + strip_v: false + default: v0.0.0 - name: Build Docker Image run: .github/jobs/build_docker_image.sh env: - GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag_name }} + GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag }} - name: Push Docker Image run: .github/jobs/push_docker_image.sh From d135e14fbeab6a7716446a99cd3b74b7f2cd2098 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Aug 2023 13:28:53 -0600 Subject: [PATCH 04/17] Per #12, add cmake to the list of packages to be installed. This is needed to compile Proj from scratch --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eb5f6c0..44b4055 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ ENV MET_FONT_DIR /usr/local/share/met/fonts # Install required system tools RUN apt update && apt -y upgrade \ - && apt install -y build-essential gfortran wget unzip curl imagemagick \ + && apt install -y build-essential cmake gfortran wget unzip curl imagemagick \ libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ emacs less \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev \ From 2e09724f62ee4ca873e21ba70efcbd1902c9c697 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Aug 2023 13:31:40 -0600 Subject: [PATCH 05/17] Per #12, temporary change to the Dockerfile to pull compile_MET_all.sh script from the feature_2669_proj branch to demonstrate that this that script runs fine via GHA. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 44b4055..3745bca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,8 +64,8 @@ RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="cod WORKDIR /met RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ + && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/environment/development.docker \ && tar -zxf tar_files.tgz \ && export SKIP_MET=yes \ && chmod +x compile_MET_all.sh \ From 5f9f1f3fcd8fd2684a1f980d4b6295cd4fb222fa Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 25 Aug 2023 13:55:23 -0600 Subject: [PATCH 06/17] Per #12, the proj library requires cmake and the tiff library to compile. --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3745bca..2bf1b44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,11 +27,12 @@ ENV MET_FONT_DIR /usr/local/share/met/fonts # Install required system tools RUN apt update && apt -y upgrade \ - && apt install -y build-essential cmake gfortran wget unzip curl imagemagick \ + && apt install -y build-essential gfortran wget unzip curl imagemagick \ libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ emacs less \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev \ - libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev + libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ + cmake libtiff-dev # # Install Python From d8eea4b843b65e9833563bd9fcfe4cb5f1e6298d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 13:49:01 -0600 Subject: [PATCH 07/17] Per #12, also install sqlite3 which is needed by cmake --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2bf1b44..b80e1ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,9 @@ RUN apt update && apt -y upgrade \ && apt install -y build-essential gfortran wget unzip curl imagemagick \ libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ emacs less \ - libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev \ + libreadline-gplv2-dev libncursesw5-dev libssl-dev tk-dev \ libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ - cmake libtiff-dev + cmake libtiff-dev sqlite3 libsqlite3-dev # # Install Python From fc5a98e27360a518a3022c5a0dbcc01fe09aa9ef Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 15:28:05 -0600 Subject: [PATCH 08/17] Per #12, remove default option from the devops-actions/action-get-tag step --- .github/workflows/build_and_push_docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_push_docker.yml b/.github/workflows/build_and_push_docker.yml index a582c1e..103a5ec 100644 --- a/.github/workflows/build_and_push_docker.yml +++ b/.github/workflows/build_and_push_docker.yml @@ -22,7 +22,6 @@ jobs: uses: devops-actions/action-get-tag@v1.0.1 with: strip_v: false - default: v0.0.0 - name: Build Docker Image run: .github/jobs/build_docker_image.sh From 445fb78e167842072ea3cb77f63b666788a816a2 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 16:02:47 -0600 Subject: [PATCH 09/17] Per #12, update action-get-tag from v1.0.1 to v1.0.2 to get past the set-output warning message. --- .github/workflows/build_and_push_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_push_docker.yml b/.github/workflows/build_and_push_docker.yml index 103a5ec..302bbdc 100644 --- a/.github/workflows/build_and_push_docker.yml +++ b/.github/workflows/build_and_push_docker.yml @@ -19,7 +19,7 @@ jobs: - name: Get tag name id: get_tag_name - uses: devops-actions/action-get-tag@v1.0.1 + uses: devops-actions/action-get-tag@v1.0.2 with: strip_v: false From 228daed722f1f30b65ac8ad219720d8568d180fa Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 16:07:15 -0600 Subject: [PATCH 10/17] Per #12, fix workflow to reference tag instead of tag_name --- .github/workflows/build_and_push_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_push_docker.yml b/.github/workflows/build_and_push_docker.yml index 302bbdc..13b7247 100644 --- a/.github/workflows/build_and_push_docker.yml +++ b/.github/workflows/build_and_push_docker.yml @@ -31,7 +31,7 @@ jobs: - name: Push Docker Image run: .github/jobs/push_docker_image.sh env: - GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag_name }} + GITHUB_TAG: ${{ steps.get_tag_name.outputs.tag }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} From f9982d7d5f273ad246516f4ba3ba9b0eb05ca02b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 17:00:41 -0600 Subject: [PATCH 11/17] Per #12, update to checkout@v3 action to get past the node12 GHA warning. --- .github/workflows/build_and_push_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_push_docker.yml b/.github/workflows/build_and_push_docker.yml index 13b7247..69fcfbf 100644 --- a/.github/workflows/build_and_push_docker.yml +++ b/.github/workflows/build_and_push_docker.yml @@ -15,7 +15,7 @@ jobs: name: Handle Docker Images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get tag name id: get_tag_name From 399b69bfc03d15d925917c7a05adf2e0dcc72461 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 21:53:20 -0600 Subject: [PATCH 12/17] Update Dockerfile to pull from develop rather than my feature branch --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b80e1ee..cd5f70a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,8 @@ RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="cod WORKDIR /met RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/environment/development.docker \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ && tar -zxf tar_files.tgz \ && export SKIP_MET=yes \ && chmod +x compile_MET_all.sh \ From ff41c0dda58c9397c0b090b1ae1b0a7a75ccd801 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 28 Aug 2023 21:59:10 -0600 Subject: [PATCH 13/17] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 541aaa0..ffaced8 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ Please see the [MET website](https://dtcenter.org/community-code/model-evaluatio Version History =============== +v2.1 +---- + +* Completes transition to debian base image +* Adds Proj library dependency + v2.0_debian10 ------------- From bf2a1cffb145ef8ed07b918a0373432c39b63717 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 29 Aug 2023 15:17:50 -0600 Subject: [PATCH 14/17] Per #12, try moving python3.10 compilation after compile_MET_all.sh compiles NetCDF and HDF5 --- Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd5f70a..4907aec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,19 +34,6 @@ RUN apt update && apt -y upgrade \ libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ cmake libtiff-dev sqlite3 libsqlite3-dev -# -# Install Python -# https://linuxhint.com/install-python-debian-10/ -# -RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ - && tar xzf Python-${PYTHON_VER}.tgz \ - && cd Python-${PYTHON_VER} \ - && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ - && make -j `nproc` \ - && make install \ - && python3 -m pip install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ && mkdir -p /usr/local/share/met \ && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met @@ -65,9 +52,22 @@ RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="cod WORKDIR /met RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ + && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/environment/development.docker \ && tar -zxf tar_files.tgz \ && export SKIP_MET=yes \ && chmod +x compile_MET_all.sh \ && ./compile_MET_all.sh development.docker + +# +# Install Python +# https://linuxhint.com/install-python-debian-10/ +# +RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ + && tar xzf Python-${PYTHON_VER}.tgz \ + && cd Python-${PYTHON_VER} \ + && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ + && make -j `nproc` \ + && make install \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install numpy xarray netCDF4 From 0fcdb655d361dfa9392f32f55d83fb9199cd88c8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 1 Sep 2023 14:15:05 -0600 Subject: [PATCH 15/17] Per #2669, update to Debian realease 12 (bookworm) since RAL-IT is planning to skip directly from Debian 10 to 12. --- Dockerfile | 64 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4907aec..6281534 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -ARG DEBIAN_VERSION=10 +ARG DEBIAN_VERSION=12 FROM debian:${DEBIAN_VERSION}-slim MAINTAINER George McCabe # -# Define the compilers. +# Define the compilers # ENV CC /usr/bin/gcc ENV CXX /usr/bin/g++ @@ -15,7 +15,7 @@ ENV PYTHON_VER 3.10.4 ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz # -# Setup the environment for interactive bash shell. +# Set up the environment for interactive bash shell # Set soft limit to unlimited to prevent GRIB2 seg faults # RUN echo export MET_BASE=/usr/local/share/met >> /root/.bashrc \ @@ -25,49 +25,57 @@ RUN echo export MET_BASE=/usr/local/share/met >> /root/.bashrc \ ENV MET_FONT_DIR /usr/local/share/met/fonts +# # Install required system tools +# RUN apt update && apt -y upgrade \ - && apt install -y build-essential gfortran wget unzip curl imagemagick \ - libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ - emacs less \ - libreadline-gplv2-dev libncursesw5-dev libssl-dev tk-dev \ - libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ - cmake libtiff-dev sqlite3 libsqlite3-dev + && apt install -y build-essential gfortran wget unzip curl imagemagick \ + libcurl4-gnutls-dev m4 git automake flex bison libjpeg-dev libpixman-1-dev \ + emacs vim less \ + libreadline-dev libncursesw5-dev libssl-dev tk-dev \ + libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ + cmake libtiff-dev sqlite3 libsqlite3-dev RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ && mkdir -p /usr/local/share/met \ && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met +# # Fix rules for ghostscript files in convert # See: https://en.linuxportal.info/tutorials/troubleshooting/how-to-fix-errors-from-imagemagick-imagick-conversion-system-security-policy # RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="coder" rights="read | write" pattern="PS/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="EPS"/policy domain="coder" rights="read | write" pattern="EPS"/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read | write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml \ - && sed -i 's/policy domain="coder" rights="none" pattern="XPS"/policy domain="coder" rights="read | write" pattern="XPS"/g' /etc/ImageMagick-6/policy.xml + && sed -i 's/policy domain="coder" rights="none" pattern="EPS"/policy domain="coder" rights="read | write" pattern="EPS"/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="PDF"/policy domain="coder" rights="read | write" pattern="PDF"/g' /etc/ImageMagick-6/policy.xml \ + && sed -i 's/policy domain="coder" rights="none" pattern="XPS"/policy domain="coder" rights="read | write" pattern="XPS"/g' /etc/ImageMagick-6/policy.xml # -# Set the working directory. +# Set the working directory # WORKDIR /met RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/environment/development.docker \ - && tar -zxf tar_files.tgz \ - && export SKIP_MET=yes \ - && chmod +x compile_MET_all.sh \ - && ./compile_MET_all.sh development.docker + && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/environment/development.docker \ + && tar -zxf tar_files.tgz \ + && export SKIP_MET=TRUE \ + && chmod +x compile_MET_all.sh \ + && ./compile_MET_all.sh development.docker # -# Install Python -# https://linuxhint.com/install-python-debian-10/ +# Install Python from source # RUN wget https://www.python.org/ftp/python/${PYTHON_VER}/Python-${PYTHON_VER}.tgz \ - && tar xzf Python-${PYTHON_VER}.tgz \ - && cd Python-${PYTHON_VER} \ - && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ - && make -j `nproc` \ - && make install \ - && python3 -m pip install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 + && tar xzf Python-${PYTHON_VER}.tgz \ + && cd Python-${PYTHON_VER} \ + && ./configure --enable-optimizations --enable-shared LDFLAGS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib" \ + && make -j `nproc` \ + && make install \ + && ln -s /usr/local/bin/python3 /usr/local/bin/python \ + && BLDOPTS="--force-reinstall --global-option=build_ext --global-option=\"-R/usr/local/lib\" --global-option=\"-L/usr/local/lib\"" \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install ${BLDOPTS} numpy \ + && python3 -m pip install ${BLDOPTS} xarray \ + && export HDF5_DIR=/usr/local/ \ + && export NETCDF4_DIR=/usr/local/ \ + && python3 -m pip install ${BLDOPTS} netCDF4 From 2570e4d8978a8979c05984cacb18cc0090d50c92 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 5 Sep 2023 11:56:26 -0600 Subject: [PATCH 16/17] Update README.md Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffaced8..f40fdee 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Version History v2.1 ---- -* Completes transition to debian base image +* Completes transition to debian base image using Debian 12 (bookworm) * Adds Proj library dependency v2.0_debian10 From 6005ac5e6c10d51e79c253e432dc024b96d59fff Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 5 Sep 2023 13:30:26 -0600 Subject: [PATCH 17/17] Per #12, after dtcenter/MET#2672 was merged into develop, update the Dockerfile here to pull compile_MET_all.sh from develop rather than the feature_2669_proj feature branch. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6281534..346d116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,8 +55,8 @@ RUN sed -i 's/policy domain="coder" rights="none" pattern="PS/policy domain="cod WORKDIR /met RUN wget https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/tar_files.tgz \ - && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/installation/compile_MET_all.sh \ - && wget https://raw.githubusercontent.com/dtcenter/MET/feature_2669_proj/internal/scripts/environment/development.docker \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/installation/compile_MET_all.sh \ + && wget https://raw.githubusercontent.com/dtcenter/MET/develop/internal/scripts/environment/development.docker \ && tar -zxf tar_files.tgz \ && export SKIP_MET=TRUE \ && chmod +x compile_MET_all.sh \