Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement shared scripts in WGCW image #816

Merged
merged 3 commits into from
Jul 22, 2024
Merged

Implement shared scripts in WGCW image #816

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a66cf1
Implement shared scripts in WGCW image
ianpittwood Jul 17, 2024
23429ee
Wrap python install with apt update/clean
ianpittwood Jul 19, 2024
ca20dcd
Merge branch 'dev' into wgcw-posit-scripts
ianpittwood Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docker-bake.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ variable WORKBENCH_BUILD_MATRIX {
variable WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX {
default = {
builds = [
{os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.11.9", py_alternate = "3.10.14"},
{os = "ubuntu2204", r_primary = "4.4.0", r_alternate = "4.3.3", py_primary = "3.12.1", py_alternate = "3.11.7"},
]
}
}
Expand Down Expand Up @@ -471,6 +471,9 @@ target "workbench-for-google-cloud-workstations" {

dockerfile = "Dockerfile.${builds.os}"
context = "workbench-for-google-cloud-workstations"
contexts = {
product-base-pro = "target:product-base-pro-${builds.os}-r${replace(builds.r_primary, ".", "-")}_${replace(builds.r_alternate, ".", "-")}-py${replace(builds.py_primary, ".", "-")}_${replace(builds.py_alternate, ".", "-")}"
}

matrix = WORKBENCH_GOOGLE_CLOUD_WORKSTATION_BUILD_MATRIX
args = {
Expand Down
87 changes: 30 additions & 57 deletions workbench-for-google-cloud-workstations/Dockerfile.ubuntu2204
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
FROM product-base-pro as posit_base
FROM us-central1-docker.pkg.dev/cloud-workstations-images/predefined/base:latest as build

### ARG declarations ###
Expand Down Expand Up @@ -28,56 +29,37 @@ ENV DIAGNOSTIC_ONLY false
ENV LICENSE_MANAGER_PATH /opt/rstudio-license
ENV WORKBENCH_JUPYTER_PATH=/usr/local/bin/jupyter

### Copy scripts from Posit Base ###
COPY --from=posit_base /opt/positscripts /opt/positscripts

### Copy package lists and install scripts ###
COPY deps/* /
COPY deps/* /tmp/

### Update/upgrade system packages ###
COPY deps/apt_packages.txt /tmp/apt_packages.txt
RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
&& apt-get update --fix-missing \
&& apt-get upgrade -yq \
&& xargs -a /apt_packages.txt apt-get install -yq --no-install-recommends \
&& rm /apt_packages.txt \
&& rm -rf /var/lib/apt/lists/*
&& ${SCRIPTS_DIR}/apt.sh --update upgrade \
&& ${SCRIPTS_DIR}/apt.sh install $(cat /tmp/apt_packages.txt) \
&& ${SCRIPTS_DIR}/apt.sh --clean \
&& rm /tmp/apt_packages.txt

### Install R versions ###
RUN curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION}_1_amd64.deb \
&& curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb \
&& apt-get update \
&& apt-get install -yq --no-install-recommends ./r-${R_VERSION}_1_amd64.deb \
&& apt-get install -yq --no-install-recommends ./r-${R_VERSION_ALT}_1_amd64.deb \
&& rm -f ./r-${R_VERSION}_1_amd64.deb \
&& rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \
&& ln -s /opt/R/${R_VERSION}/bin/R /usr/local/bin/R \
&& ln -s /opt/R/${R_VERSION}/bin/Rscript /usr/local/bin/Rscript \
&& rm -rf /var/lib/apt/lists/*
COPY deps/r_packages.txt /tmp/r_packages.txt
RUN ${SCRIPTS_DIR}/apt.sh --update \
&& R_VERSION=${R_VERSION} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \
&& R_VERSION=${R_VERSION_ALT} ${SCRIPTS_DIR}/install_r.sh -r /tmp/r_packages.txt \
&& ${SCRIPTS_DIR}/apt.sh --clean \
&& ln -s /opt/R/${R_VERSION} /opt/R/default \
&& ln -s /opt/R/default/bin/R /usr/local/bin/R \
&& ln -s /opt/R/default/bin/Rscript /usr/local/bin/Rscript \
&& rm -f /tmp/r_packages.txt

### Install Python versions ###
RUN curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION}_1_amd64.deb \
&& curl -O https://cdn.rstudio.com/python/ubuntu-2204/pkgs/python-${PYTHON_VERSION_ALT}_1_amd64.deb \
&& apt-get update \
&& apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION}_1_amd64.deb \
&& apt-get install -yq --no-install-recommends ./python-${PYTHON_VERSION_ALT}_1_amd64.deb \
&& rm -rf python-${PYTHON_VERSION}_1_amd64.deb \
&& rm -rf python-${PYTHON_VERSION_ALT}_1_amd64.deb \
&& /opt/python/${PYTHON_VERSION}/bin/python3 -m ensurepip --upgrade \
&& /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install 'virtualenv<20' \
&& /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade setuptools \
&& /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --upgrade pip \
&& /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m ensurepip --upgrade \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install 'virtualenv<20' \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade setuptools \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --upgrade pip \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \
&& rm -rf /var/lib/apt/lists/*

### Install basic data science packages for Python and R ###
RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install -r /py_packages.txt \
&& /opt/python/${PYTHON_VERSION}/bin/python3 -m pip cache purge \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install -r /py_packages.txt \
&& /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip cache purge \
&& ./install_r_packages.sh \
&& rm install_r_packages.sh py_packages.txt r_packages.txt
COPY deps/requirements.txt /tmp/requirements.txt
RUN PYTHON_VERSION=${PYTHON_VERSION} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \
&& PYTHON_VERSION=${PYTHON_VERSION_ALT} ${SCRIPTS_DIR}/install_python.sh -r /tmp/requirements.txt \
&& ln -s /opt/python/${PYTHON_VERSION} /opt/python/default \
&& rm -f /tmp/requirements.txt
ianpittwood marked this conversation as resolved.
Show resolved Hide resolved

### Locale configuration ###
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
Expand All @@ -89,13 +71,10 @@ ENV LC_ALL en_US.UTF-8
RUN ln -s /lib/rstudio-server/bin/quarto/bin/quarto /usr/local/bin/quarto

### Install Pro Drivers ###
RUN apt-get update \
&& apt-get install -yq --no-install-recommends unixodbc unixodbc-dev \
&& curl -O https://cdn.rstudio.com/drivers/7C152C12/installer/rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \
&& apt-get update \
&& apt-get install -yq --no-install-recommends ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \
&& rm -f ./rstudio-drivers_${DRIVERS_VERSION}_amd64.deb \
&& rm -rf /var/lib/apt/lists/* \
RUN ${SCRIPTS_DIR}/apt.sh --update upgrade \
&& ${SCRIPTS_DIR}/apt.sh install unixodbc unixodbc-dev \
&& DRIVERS_VERSION=${DRIVERS_VERSION} ${SCRIPTS_DIR}/install_drivers.sh \
&& ${SCRIPTS_DIR}/apt.sh --clean \
&& cp /opt/rstudio-drivers/odbcinst.ini.sample /etc/odbcinst.ini \
&& /opt/R/${R_VERSION}/bin/R -e 'install.packages("odbc", repos="https://packagemanager.rstudio.com/cran/__linux__/jammy/latest")'

Expand All @@ -110,6 +89,7 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION
&& apt-get update \
&& apt-get install -y --no-install-recommends ./rstudio-workbench.deb \
&& rm ./rstudio-workbench.deb \
&& apt-get remove -yq dpkg-sig \
&& apt-get autoremove -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
Expand All @@ -119,7 +99,7 @@ RUN curl -o rstudio-workbench.deb "${RSW_DOWNLOAD_URL}/${RSW_NAME}-${RSW_VERSION
# TODO(ianpittwood): Replace monitor download with $RSW_VERSION after upgrading to 2023.06.0
RUN mkdir -p /opt/rstudio-license/ \
&& mkdir -p /var/lib/rstudio-workbench \
&& curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-2023.06.0-419.pro1.tar.gz" | \
&& curl -sL "https://s3.amazonaws.com/rstudio-ide-build/monitor/jammy/rsp-monitor-workbench-gcpw-amd64-${RSW_VERSION//+/-}.tar.gz" | \
ianpittwood marked this conversation as resolved.
Show resolved Hide resolved
tar xzvf - --strip 2 -C /opt/rstudio-license/ \
&& chmod 0755 /opt/rstudio-license/license-manager \
&& mv /opt/rstudio-license/license-manager /opt/rstudio-license/license-manager-orig \
Expand Down Expand Up @@ -166,12 +146,5 @@ COPY conf/* /etc/rstudio/
COPY --chmod=755 workstation-startup/* /etc/workstation-startup.d/
COPY --chmod=644 jupyter/jupyter_notebook_config.json /opt/python/jupyter/etc/jupyter/jupyter_notebook_config.json

### Clean up ###
RUN apt-get remove -yq dpkg-sig \
&& apt-get install -yqf --no-install-recommends \
&& apt-get autoremove -yq \
&& apt-get clean -yq \
&& rm -rf /var/lib/apt/lists/*

EXPOSE 80/tcp
EXPOSE 5559/tcp
10 changes: 0 additions & 10 deletions workbench-for-google-cloud-workstations/deps/install_r_packages.sh
View file
Open in desktop

This file was deleted.

0 ...e-cloud-workstations/deps/py_packages.txt → ...-cloud-workstations/deps/requirements.txt
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion workbench-for-google-cloud-workstations/test/goss.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ command:
]
{{ $python_version := .Env.PYTHON_VERSION }}
{{ $python_version_alt := .Env.PYTHON_VERSION_ALT }}
{{ $py_package_list := readFile "/tmp/deps/py_packages.txt" | splitList "\n" }}
{{ $py_package_list := readFile "/tmp/deps/requirements.txt" | splitList "\n" }}
{{- range $py_package_list }}
Check Python {{ $python_version }} has "{{.}}" installed:
exec: /opt/python/{{$python_version}}/bin/pip show {{.}}
Expand Down
Loading