From 3a66cf1ed090655f7481aa0045e8b4a556045ebf Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 17 Jul 2024 10:00:04 -0600 Subject: [PATCH 1/2] Implement shared scripts in WGCW image --- docker-bake.hcl | 5 +- .../Dockerfile.ubuntu2204 | 87 +++++++------------ .../deps/install_r_packages.sh | 10 --- .../{py_packages.txt => requirements.txt} | 0 .../test/goss.yaml | 2 +- 5 files changed, 35 insertions(+), 69 deletions(-) delete mode 100755 workbench-for-google-cloud-workstations/deps/install_r_packages.sh rename workbench-for-google-cloud-workstations/deps/{py_packages.txt => requirements.txt} (100%) diff --git a/docker-bake.hcl b/docker-bake.hcl index 86729c29..bf9957c3 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -184,7 +184,7 @@ variable WORKBENCH_BUILD_MATRIX { variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX { default = { builds = [ - {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.11.9", py_alternate = "3.10.14"}, + {os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.12.1", py_alternate = "3.11.7"}, ] } } @@ -471,6 +471,9 @@ target "workbench-for-google-cloud-workstations" { dockerfile = "Dockerfile.${builds.os}" context = "workbench-for-google-cloud-workstations" + contexts = { + product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}" + } matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX args = { diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 index e66c3c70..8c821288 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 @@ -1,3 +1,4 @@ +FROM product-base-pro as posit_base FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest as build ### ARG declarations ### @@ -28,56 +29,37 @@ ENV DIAGNOSTIC_ONLY false ENV LICENSE_MANAGER_PATH /opt/rstudio-license ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter +### Copy scripts from Posit Base ### +COPY --from=posit_base /opt/positscripts /opt/positscripts + ### Copy package lists and install scripts ### -COPY deps/* / +COPY deps/* /tmp/ ### Update/upgrade system packages ### +COPY deps/apt_packages.txt /tmp/apt_packages.txt RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \ - && apt-get update --fix-missing \ - && apt-get upgrade -yq \ - && xargs -a /apt_packages.txt apt-get install -yq --no-install-recommends \ - && rm /apt_packages.txt \ - && rm -rf /var/lib/apt/lists/* + && ${SCRIPTS_DIR}/apt.sh --update upgrade \ + && ${SCRIPTS_DIR}/apt.sh install $(cat /tmp/apt_packages.txt) \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && rm /tmp/apt_packages.txt ### Install R versions ### -RUN curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \ - && rm -f ./r-${R_VERSION}_1_amd64.deb \ - && rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \ - && ln -s /opt/R/${R_VERSION}/bin/R /usr/local/bin/R \ - && ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript \ - && rm -rf /var/lib/apt/lists/* +COPY deps/r_packages.txt /tmp/r_packages.txt +RUN ${SCRIPTS_DIR}/apt.sh --update \ + && R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \ + && R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \ + && ${SCRIPTS_DIR}/apt.sh --clean \ + && ln -s /opt/R/${R_VERSION} /opt/R/default \ + && ln -s /opt/R/default/bin/R /usr/local/bin/R \ + && ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript \ + && rm -f /tmp/r_packages.txt ### Install Python versions ### -RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \ - && curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \ - && apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && rm -rf python-${PYTHON_VERSION}_1_amd64.deb \ - && rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m ensurepip --upgrade \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade pip \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m ensurepip --upgrade \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ - && rm -rf /var/lib/apt/lists/* - -### Install basic data science packages for Python and R ### -RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /py_packages.txt \ - && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install -r /py_packages.txt \ - && /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \ - && ./install_r_packages.sh \ - && rm install_r_packages.sh py_packages.txt r_packages.txt +COPY deps/requirements.txt /tmp/requirements.txt +RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \ + && rm -f /tmp/requirements.txt ### Locale configuration ### RUN localedef -i en_US -f UTF-8 en_US.UTF-8 @@ -89,13 +71,10 @@ ENV LC_ALL en_US.UTF-8 RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto ### Install Pro Drivers ### -RUN apt-get update \ - && apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \ - && curl -O https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && apt-get update \ - && apt-get install -yq --no-install-recommends ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \ - && rm -rf /var/lib/apt/lists/* \ +RUN ${SCRIPTS_DIR}/apt.sh --update upgrade \ + && ${SCRIPTS_DIR}/apt.sh install unixodbc unixodbc-dev \ + && DRIVERS_VERSION=${DRIVERS_VERSION} ${SCRIPTS_DIR}/install_drivers.sh \ + && ${SCRIPTS_DIR}/apt.sh --clean \ && cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \ && /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/jammy/latest")' @@ -110,6 +89,7 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION && apt-get update \ && apt-get install -y --no-install-recommends ./rstudio-workbench.deb \ && rm ./rstudio-workbench.deb \ + && apt-get remove -yq dpkg-sig \ && apt-get autoremove -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ @@ -119,7 +99,7 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION # TODO(ianpittwood): Replace monitor download with $RSW_VERSION after upgrading to 2023.06.0 RUN mkdir -p /opt/rstudio-license/ \ && mkdir -p /var/lib/rstudio-workbench \ - && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-2023.06.0-419.pro1.tar.gz" | \ + && curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-${RSW_VERSION//+/-}.tar.gz" | \ tar xzvf - --strip 2 -C /opt/rstudio-license/ \ && chmod 0755 /opt/rstudio-license/license-manager \ && mv /opt/rstudio-license/license-manager /opt/rstudio-license/license-manager-orig \ @@ -166,12 +146,5 @@ COPY conf/* /etc/rstudio/ COPY --chmod=755 workstation-startup/* /etc/workstation-startup.d/ COPY --chmod=644 jupyter/jupyter_notebook_config.json /opt/python/jupyter/etc/jupyter/jupyter_notebook_config.json -### Clean up ### -RUN apt-get remove -yq dpkg-sig \ - && apt-get install -yqf --no-install-recommends \ - && apt-get autoremove -yq \ - && apt-get clean -yq \ - && rm -rf /var/lib/apt/lists/* - EXPOSE 80/tcp EXPOSE 5559/tcp diff --git a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh b/workbench-for-google-cloud-workstations/deps/install_r_packages.sh deleted file mode 100755 index 885ee27f..00000000 --- a/workbench-for-google-cloud-workstations/deps/install_r_packages.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -ex - -UBUNTU_CODENAME=$(lsb_release -cs) -CRAN_REPO="https://packagemanager.posit.co/cran/__linux__/${UBUNTU_CODENAME}/latest" - -r_packages=$(awk '{print "\"" $0 "\""}' r_packages.txt | paste -d',' -s -) -/opt/R/${R_VERSION}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" -/opt/R/${R_VERSION_ALT}/bin/R --slave -e "install.packages(c(${r_packages}), repos = \"${CRAN_REPO}\")" diff --git a/workbench-for-google-cloud-workstations/deps/py_packages.txt b/workbench-for-google-cloud-workstations/deps/requirements.txt similarity index 100% rename from workbench-for-google-cloud-workstations/deps/py_packages.txt rename to workbench-for-google-cloud-workstations/deps/requirements.txt diff --git a/workbench-for-google-cloud-workstations/test/goss.yaml b/workbench-for-google-cloud-workstations/test/goss.yaml index c55ff6d8..a978ec25 100644 --- a/workbench-for-google-cloud-workstations/test/goss.yaml +++ b/workbench-for-google-cloud-workstations/test/goss.yaml @@ -204,7 +204,7 @@ command: ] {{ $python_version := .Env.PYTHON_VERSION }} {{ $python_version_alt := .Env.PYTHON_VERSION_ALT }} - {{ $py_package_list := readFile "/tmp/deps/py_packages.txt" | splitList "\n" }} + {{ $py_package_list := readFile "/tmp/deps/requirements.txt" | splitList "\n" }} {{- range $py_package_list }} Check Python {{ $python_version }} has "{{.}}" installed: exec: /opt/python/{{$python_version}}/bin/pip show {{.}} From 23429ee676859d7da532838b81312edc0b1ea780 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 19 Jul 2024 10:02:02 -0600 Subject: [PATCH 2/2] Wrap python install with apt update/clean --- workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 | 4 +++- workbench-for-google-cloud-workstations/deps/apt_packages.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 index 8c821288..cbdfb247 100644 --- a/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 +++ b/workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204 @@ -56,8 +56,10 @@ RUN ${SCRIPTS_DIR}/apt.sh --update \ ### Install Python versions ### COPY deps/requirements.txt /tmp/requirements.txt -RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ +RUN ${SCRIPTS_DIR}/apt.sh --update \ + && PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ && PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \ + && ${SCRIPTS_DIR}/apt.sh --clean \ && ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \ && rm -f /tmp/requirements.txt diff --git a/workbench-for-google-cloud-workstations/deps/apt_packages.txt b/workbench-for-google-cloud-workstations/deps/apt_packages.txt index bf6d9c10..df9f9911 100644 --- a/workbench-for-google-cloud-workstations/deps/apt_packages.txt +++ b/workbench-for-google-cloud-workstations/deps/apt_packages.txt @@ -22,6 +22,7 @@ libuser1-dev libxext6 libxrender1 locales +lsb-release oddjob-mkhomedir rrdtool sssd