From f3c8574b196533fbc0c8aa6229a7640f8e9ccbc4 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 6 Mar 2023 06:20:01 -0300 Subject: [PATCH] #762: Update conda/ubuntu packages (#347) * Reformat scripts/update_apt_package_in_package_list.sh * Make cryptography 39.0.2 and openssl 3.0.8 working with conda by updating mamba * Update ubuntu packages * Fix package versions in install script tests Co-authored-by: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Co-authored-by: Torsten Kilias --- .../test_files/apt/with_versions/all_versions_specified | 2 +- .../test_files/apt/with_versions/some_missing_versions | 2 +- .../flavor_base/build_deps/packages/apt_get_packages | 2 +- .../flavor_base_deps/packages/apt_get_packages | 4 ++-- .../flavor_base/language_deps/packages/apt_get_packages | 2 +- .../flavor_base/build_deps/packages/apt_get_packages | 2 +- .../flavor_base_deps/packages/apt_get_packages | 2 +- .../flavor_base/language_deps/packages/apt_get_packages | 2 +- .../flavor_base/conda_deps/packages/apt_get_packages | 2 +- .../flavor_base/language_deps/packages/conda_packages | 3 ++- .../flavor_base/udfclient_deps/packages/conda_packages | 2 +- .../flavor_base/language_deps/packages/conda_packages | 3 ++- .../nvidia_driver_deps/packages/apt_get_packages | 2 +- .../flavor_base/udfclient_deps/packages/conda_packages | 2 +- scripts/update_apt_package_in_package_list.sh | 9 ++++++--- 15 files changed, 23 insertions(+), 18 deletions(-) diff --git a/ext/scripts/tests/install_scripts/test_files/apt/with_versions/all_versions_specified b/ext/scripts/tests/install_scripts/test_files/apt/with_versions/all_versions_specified index 729903e3a..7a45d6c6b 100644 --- a/ext/scripts/tests/install_scripts/test_files/apt/with_versions/all_versions_specified +++ b/ext/scripts/tests/install_scripts/test_files/apt/with_versions/all_versions_specified @@ -1,2 +1,2 @@ -curl|7.58.0-2ubuntu3.22 # t1 +curl|7.58.0-2ubuntu3.23 # t1 wget|1.19.4-1ubuntu2.2 # t2 diff --git a/ext/scripts/tests/install_scripts/test_files/apt/with_versions/some_missing_versions b/ext/scripts/tests/install_scripts/test_files/apt/with_versions/some_missing_versions index f5610fe84..ff9d952f1 100644 --- a/ext/scripts/tests/install_scripts/test_files/apt/with_versions/some_missing_versions +++ b/ext/scripts/tests/install_scripts/test_files/apt/with_versions/some_missing_versions @@ -1,3 +1,3 @@ -curl|7.58.0-2ubuntu3.22 # t1 +curl|7.58.0-2ubuntu3.23 # t1 wget| # t2 git| diff --git a/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages index b27f995f4..81428560c 100644 --- a/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages +++ b/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages @@ -1,7 +1,7 @@ coreutils|8.28-1ubuntu1 locales|2.27-3ubuntu1.6 tar|1.29b-2ubuntu0.2 -curl|7.58.0-2ubuntu3.22 +curl|7.58.0-2ubuntu3.23 openjdk-11-jdk|11.0.11+9-0ubuntu2~18.04 build-essential|12.4ubuntu1 libpcre3-dev|2:8.39-9 diff --git a/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages b/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages index fb8c0b013..ab8b9c5d7 100644 --- a/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages +++ b/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages @@ -1,5 +1,5 @@ unzip|6.0-21ubuntu1.2 wget|1.19.4-1ubuntu2.2 -git|1:2.17.1-1ubuntu0.16 -libcurl4-openssl-dev|7.58.0-2ubuntu3.22 +git|1:2.17.1-1ubuntu0.17 +libcurl4-openssl-dev|7.58.0-2ubuntu3.23 build-essential|12.4ubuntu1 diff --git a/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages b/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages index 9cbdcfae3..c31f3ba85 100644 --- a/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages +++ b/flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages @@ -1,2 +1,2 @@ -curl|7.58.0-2ubuntu3.22 +curl|7.58.0-2ubuntu3.23 ca-certificates|20211016ubuntu0.18.04.1 diff --git a/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages index fee887134..6e5eb914c 100644 --- a/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages +++ b/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/build_deps/packages/apt_get_packages @@ -1,7 +1,7 @@ coreutils|8.28-1ubuntu1 locales|2.27-3ubuntu1.6 tar|1.29b-2ubuntu0.2 -curl|7.68.0-1ubuntu2.15 +curl|7.68.0-1ubuntu2.16 openjdk-11-jdk|11.0.11+9-0ubuntu2~18.04 build-essential|12.4ubuntu1 libpcre3-dev|2:8.39-9 diff --git a/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages b/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages index f26d0447d..a64a3bd28 100644 --- a/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages +++ b/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/flavor_base_deps/packages/apt_get_packages @@ -1,4 +1,4 @@ unzip|6.0-25ubuntu1.1 git|1:2.25.1-1ubuntu3.10 -libcurl4-openssl-dev|7.68.0-1ubuntu2.15 +libcurl4-openssl-dev|7.68.0-1ubuntu2.16 build-essential|12.8ubuntu1.1 diff --git a/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages b/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages index aab4d169e..d98961daa 100644 --- a/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages +++ b/flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/apt_get_packages @@ -1,5 +1,5 @@ ca-certificates|20211016ubuntu0.20.04.1 python3.8-dev|3.8.10-0ubuntu1~20.04.6 python3-distutils|3.8.10-0ubuntu1~20.04 -curl|7.68.0-1ubuntu2.15 +curl|7.68.0-1ubuntu2.16 diff --git a/flavors/template-Exasol-all-python-3.8-conda/flavor_base/conda_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.8-conda/flavor_base/conda_deps/packages/apt_get_packages index b5c2c2a16..b38e195ca 100644 --- a/flavors/template-Exasol-all-python-3.8-conda/flavor_base/conda_deps/packages/apt_get_packages +++ b/flavors/template-Exasol-all-python-3.8-conda/flavor_base/conda_deps/packages/apt_get_packages @@ -1,4 +1,4 @@ coreutils|8.30-3ubuntu2 locales|2.31-0ubuntu9.9 -curl|7.68.0-1ubuntu2.15 +curl|7.68.0-1ubuntu2.16 ca-certificates|20211016ubuntu0.20.04.1 diff --git a/flavors/template-Exasol-all-python-3.8-conda/flavor_base/language_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.8-conda/flavor_base/language_deps/packages/conda_packages index 88d6a2cbe..1552d9e52 100644 --- a/flavors/template-Exasol-all-python-3.8-conda/flavor_base/language_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.8-conda/flavor_base/language_deps/packages/conda_packages @@ -2,5 +2,6 @@ python|3.8.13 numpy|1.22.3 pandas|1.4.2 libblas|3.9.0=15_linux64_mkl -mamba|0.24.0 +mamba|1.3.1 ld_impl_linux-64|2.36.1 +cryptography|39.0.2 diff --git a/flavors/template-Exasol-all-python-3.8-conda/flavor_base/udfclient_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.8-conda/flavor_base/udfclient_deps/packages/conda_packages index c61ae1b73..0816aa095 100644 --- a/flavors/template-Exasol-all-python-3.8-conda/flavor_base/udfclient_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.8-conda/flavor_base/udfclient_deps/packages/conda_packages @@ -1,6 +1,6 @@ libprotobuf|3.20.1 zeromq|4.3.4 -openssl|1.1.1o +openssl|3.0.8 nss|3.77 cppzmq|4.8.1 _openmp_mutex|4.5=2_kmp_llvm # We need this, such that we can later install libblas with mkl, which is for example needed by pytorch diff --git a/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/language_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/language_deps/packages/conda_packages index 88d6a2cbe..1552d9e52 100644 --- a/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/language_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/language_deps/packages/conda_packages @@ -2,5 +2,6 @@ python|3.8.13 numpy|1.22.3 pandas|1.4.2 libblas|3.9.0=15_linux64_mkl -mamba|0.24.0 +mamba|1.3.1 ld_impl_linux-64|2.36.1 +cryptography|39.0.2 diff --git a/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/nvidia_driver_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/nvidia_driver_deps/packages/apt_get_packages index b5c2c2a16..b38e195ca 100644 --- a/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/nvidia_driver_deps/packages/apt_get_packages +++ b/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/nvidia_driver_deps/packages/apt_get_packages @@ -1,4 +1,4 @@ coreutils|8.30-3ubuntu2 locales|2.31-0ubuntu9.9 -curl|7.68.0-1ubuntu2.15 +curl|7.68.0-1ubuntu2.16 ca-certificates|20211016ubuntu0.20.04.1 diff --git a/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages index c61ae1b73..0816aa095 100644 --- a/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages @@ -1,6 +1,6 @@ libprotobuf|3.20.1 zeromq|4.3.4 -openssl|1.1.1o +openssl|3.0.8 nss|3.77 cppzmq|4.8.1 _openmp_mutex|4.5=2_kmp_llvm # We need this, such that we can later install libblas with mkl, which is for example needed by pytorch diff --git a/scripts/update_apt_package_in_package_list.sh b/scripts/update_apt_package_in_package_list.sh index 4107112bd..671861138 100644 --- a/scripts/update_apt_package_in_package_list.sh +++ b/scripts/update_apt_package_in_package_list.sh @@ -11,8 +11,11 @@ REPLACE=$3 if [[ "$REPLACE" == "yes" ]] then - SED_REPLACE_OPTION="-i" + SED_REPLACE_OPTION=("-i") else - SED_REPLACE_OPTION="" + SED_REPLACE_OPTION=() fi -grep -E -R "^$PACKAGE\|" "$FLAVOR" | cut -f 1 -d ":" | xargs -n 1 sed "$SED_REPLACE_OPTION" -E "s/^($PACKAGE)\|[^# ]*( *#.*)?$/$PACKAGE|$CANDIDATE_VERSION/g" | grep -E "^$PACKAGE\|" +grep -E -R "^$PACKAGE\|" "$FLAVOR" \ + | cut -f 1 -d ":" \ + | xargs -I{} sed "${SED_REPLACE_OPTION=[@]}" -E "s/^($PACKAGE)\|.*$/$PACKAGE|$CANDIDATE_VERSION/g" "{}" \ + | grep -E "^$PACKAGE\|"