From 9575c6dd8b4771c06fedf1b1ceedfe9e4fa38a49 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Tue, 10 Sep 2019 21:20:47 +0200 Subject: [PATCH] enh: images published on Docker Hub are now tagged with release versions --- Makefile | 32 +++++++++---------- README.md | 2 +- workflow/deploy/device-query/skaffold.yaml | 1 - workflow/deploy/device-query/src/Dockerfile | 6 ++++ workflow/deploy/jupyter/skaffold.yaml | 1 - workflow/deploy/jupyter/src/Dockerfile | 6 ++++ workflow/deploy/ml-base/skaffold.yaml | 1 - workflow/deploy/ml-base/src/Dockerfile | 7 +++- .../tensorflow-serving-base/skaffold.yaml | 1 - .../tensorflow-serving-base/src/Dockerfile | 11 +++++-- .../deploy/tensorflow-serving/skaffold.yaml | 1 - .../deploy/tensorflow-serving/src/Dockerfile | 7 ++++ workflow/deploy/tools/builder | 11 ++++--- workflow/deploy/tools/builderx | 8 +++-- workflow/deploy/tools/publish | 20 +++++++++--- 15 files changed, 78 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index a4f64b3..b53ea70 100644 --- a/Makefile +++ b/Makefile @@ -227,7 +227,7 @@ ml-base-build-and-test: ## Build, push and test ml base image for Docker on Jets workflow/deploy/tools/container-structure-test ml-base ml-base-publish: ## Publish latest ml base image on Jetson device to Docker Hub given credentials in .docker-hub.auth - workflow/deploy/tools/publish ml-base $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) + workflow/deploy/tools/publish ml-base $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) $(filter-out $@,$(MAKECMDGOALS)) device-query-build-and-test: ## Build and test device-query @@ -258,7 +258,7 @@ device-query-dev-docker-hub-parent: ## Enter build, deploy, tail, watch cycle fo cd workflow/deploy/device-query && skaffold dev -p docker-hub-parent device-query-publish: ## Publish latest device query image on Jetson device to Docker Hub given credentials in .docker-hub.auth - workflow/deploy/tools/publish device-query $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) + workflow/deploy/tools/publish device-query $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) $(filter-out $@,$(MAKECMDGOALS)) device-query-delete: ## Delete device query deployment cd workflow/deploy/device-query && skaffold delete @@ -298,7 +298,7 @@ jupyter-dev-docker-hub-parent: ## Enter build, deploy, tail, watch cycle for jup cd workflow/deploy/jupyter && skaffold dev -p docker-hub-parent jupyter-publish: ## Publish latest jupyter image on Jetson device to Docker Hub given credentials in .docker-hub.auth - workflow/deploy/tools/publish jupyter $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) + workflow/deploy/tools/publish jupyter $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) $(filter-out $@,$(MAKECMDGOALS)) jupyter-delete: ## Delete jupyter deployment cd workflow/deploy/jupyter && skaffold delete @@ -311,7 +311,7 @@ tensorflow-serving-base-build-and-test: ## Build, push and test ml tensorflow-se workflow/deploy/tools/container-structure-test tensorflow-serving-base tensorflow-serving-base-publish: ## Publish latest tensorflow-serving base image on Jetson device to Docker Hub given credentials in .docker-hub.auth - workflow/deploy/tools/publish tensorflow-serving-base $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) + workflow/deploy/tools/publish tensorflow-serving-base $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) $(filter-out $@,$(MAKECMDGOALS)) @@ -368,7 +368,7 @@ tensorflow-serving-dev-docker-hub-parent: ## Enter build, deploy, tail, watch cy cd workflow/deploy/jupyter && skaffold dev -p docker-hub-parent tensorflow-serving-publish: ## Publish latest tensorflow-serving image on Jetson device to Docker Hub given credentials in .docker-hub.auth - workflow/deploy/tools/publish tensorflow-serving $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) + workflow/deploy/tools/publish tensorflow-serving $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) $(filter-out $@,$(MAKECMDGOALS)) tensorflow-serving-delete: ## Delete tensorflow-serving deployment cd workflow/deploy/tensorflow-serving && skaffold delete @@ -394,20 +394,20 @@ l4t-dev: ## Enter cross-build, deploy, tail, watch cycle for l4t cd workflow/deploy/l4t && skaffold dev l4t-publish: ## Publish latest lt4 image on Jetson device to Docker Hub given credentials in .docker-hub.auth - workflow/deploy/tools/publish l4t $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) + workflow/deploy/tools/publish l4t $(shell sed '1q;d' .docker-hub.auth) $(shell sed '2q;d' .docker-hub.auth) $(filter-out $@,$(MAKECMDGOALS)) l4t-delete: ## Delete l4t deployment cd workflow/deploy/l4t && skaffold delete kubectl delete namespace jetson-l4t || true publish-all: ## Publish all images to DockerHub - make ml-base-publish - make device-query-publish - make jupyter-publish - make tensorflow-serving-base-publish - make tensorflow-serving-publish - JETSON_MODEL=xavier make ml-base-publish - JETSON_MODEL=xavier make device-query-publish - JETSON_MODEL=xavier make jupyter-publish - JETSON_MODEL=xavier make tensorflow-serving-base-publish - JETSON_MODEL=xavier make tensorflow-serving-publish + JETSON_MODEL=nano make ml-base-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=nano make device-query-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=nano make jupyter-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=nano make tensorflow-serving-base-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=nano make tensorflow-serving-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=xavier make ml-base-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=xavier make device-query-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=xavier make jupyter-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=xavier make tensorflow-serving-base-publish $(filter-out $@,$(MAKECMDGOALS)) + JETSON_MODEL=xavier make tensorflow-serving-publish $(filter-out $@,$(MAKECMDGOALS)) diff --git a/README.md b/README.md index 60927fb..8723b91 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Hints: - [x] optional: Semi-automatically setup USB 3.0 / **SSD boot drive** given existing installation on micro SD card on Jetson Nanos - [ ] optional: Automatically **[cross-build](https://engineering.docker.com/2019/04/multi-arch-images/) arm64 Docker images on macOS** for Jetson devices using [**buildkit**](https://github.com/moby/buildkit) and [**buildx**](https://github.com/docker/buildx) - [ ] optional: Automatically setup **firewall** on host using [`ufw`](https://wiki.ubuntu.com/UncomplicatedFirewall) for basic security -- [x] community: Publish images on [**Docker Hub**](https://hub.docker.com/u/helmuthva) and provide Skaffold profiles to pull from there instead of having to build before deploy +- [x] community: Publish versioned images on [**Docker Hub**](https://hub.docker.com/u/helmuthva) per release and provide Skaffold profiles to pull from Docker Hub instead of having to build images before deploy in Kubernetes cluster - [ ] community: Author a series of **blog** posts explaining how to set up ML in Kubernetes on Jetson devices based on this starter - [ ] ml: Deploy **Polarize.AI** ml training and inference tiers on Jetson nodes (separate project) diff --git a/workflow/deploy/device-query/skaffold.yaml b/workflow/deploy/device-query/skaffold.yaml index b3b82dd..af7d954 100644 --- a/workflow/deploy/device-query/skaffold.yaml +++ b/workflow/deploy/device-query/skaffold.yaml @@ -1,7 +1,6 @@ apiVersion: skaffold/v1beta11 kind: Config build: - local: {} tagPolicy: sha256: {} artifacts: diff --git a/workflow/deploy/device-query/src/Dockerfile b/workflow/deploy/device-query/src/Dockerfile index b4d3ac2..58ff5a2 100644 --- a/workflow/deploy/device-query/src/Dockerfile +++ b/workflow/deploy/device-query/src/Dockerfile @@ -3,6 +3,9 @@ FROM ${FROM} MAINTAINER Helmut Hoffer von Ankershoffen +# Inc on updates of base image +ENV BUMP=1 + # Build and install device query RUN cd /usr/local/cuda/samples/1_Utilities/deviceQuery && \ make clean && \ @@ -12,3 +15,6 @@ RUN cd /usr/local/cuda/samples/1_Utilities/deviceQuery && \ # Execute deviceQuery to validate cuda and driver access on boot than sleep CMD exec /bin/bash -c "trap : TERM INT; /app/deviceQuery; echo \"Sleeping until terminated so you can see the log - press ctrl+c...\"; sleep infinity & wait" + +RUN mkdir -p /meta && \ + date +"%FT%T%Z" > /meta/device-query.build diff --git a/workflow/deploy/jupyter/skaffold.yaml b/workflow/deploy/jupyter/skaffold.yaml index 2a58a62..d894f74 100644 --- a/workflow/deploy/jupyter/skaffold.yaml +++ b/workflow/deploy/jupyter/skaffold.yaml @@ -1,7 +1,6 @@ apiVersion: skaffold/v1beta11 kind: Config build: - local: {} tagPolicy: sha256: {} artifacts: diff --git a/workflow/deploy/jupyter/src/Dockerfile b/workflow/deploy/jupyter/src/Dockerfile index 4d98abe..f10127c 100644 --- a/workflow/deploy/jupyter/src/Dockerfile +++ b/workflow/deploy/jupyter/src/Dockerfile @@ -3,6 +3,9 @@ FROM ${FROM} MAINTAINER Helmut Hoffer von Ankershoffen +# Inc on updates of base image +ENV BUMP=1 + # Install Jupyter RUN conda install -y jupyter && \ conda install -y ipywidgets @@ -30,3 +33,6 @@ CMD ["jupyter", "notebook", "--allow-root", "--no-browser" ] # Expose jupyter and tensorboard EXPOSE 8888 EXPOSE 6006 + +RUN mkdir -p /meta && \ + date +"%FT%T%Z" > /meta/jupyter.build diff --git a/workflow/deploy/ml-base/skaffold.yaml b/workflow/deploy/ml-base/skaffold.yaml index 3630f14..e0c7ba1 100644 --- a/workflow/deploy/ml-base/skaffold.yaml +++ b/workflow/deploy/ml-base/skaffold.yaml @@ -1,7 +1,6 @@ apiVersion: skaffold/v1beta11 kind: Config build: - local: {} tagPolicy: sha256: {} artifacts: diff --git a/workflow/deploy/ml-base/src/Dockerfile b/workflow/deploy/ml-base/src/Dockerfile index e8b2104..0260f38 100644 --- a/workflow/deploy/ml-base/src/Dockerfile +++ b/workflow/deploy/ml-base/src/Dockerfile @@ -6,6 +6,9 @@ MAINTAINER Helmut Hoffer von Ankershoffen # Set non interactive frontend during build ENV DEBIAN_FRONTEND=noninteractive +# Simulate runlevel 1 needed on installing some packagess +ENV RUNLEVEL=1 + # Set locale and lang ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 @@ -22,7 +25,6 @@ ENV LD_LIBRARY_PATH="/usr/local/cuda-10.0/lib64:${LD_LIBRARY_PATH}" ENV LD_LIBRARY_PATH="/usr/local/lib/:${LD_LIBRARY_PATH}" ENV LD_LIBRARY_PATH="/opt/archiconda3/lib:${LD_LIBRARY_PATH}" -# ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn # Setup capabilties for cuda @@ -157,3 +159,6 @@ RUN pip install --no-cache-dir https://developer.download.nvidia.com/compute/red # Remove repository reference so child images do not have to bind-mount again RUN rm /etc/apt/sources.list.d/cuda.list + +RUN mkdir -p /meta && \ + date +"%FT%T%Z" > /meta/ml-base.build diff --git a/workflow/deploy/tensorflow-serving-base/skaffold.yaml b/workflow/deploy/tensorflow-serving-base/skaffold.yaml index bfd61c9..e6e79e3 100644 --- a/workflow/deploy/tensorflow-serving-base/skaffold.yaml +++ b/workflow/deploy/tensorflow-serving-base/skaffold.yaml @@ -1,7 +1,6 @@ apiVersion: skaffold/v1beta11 kind: Config build: - local: {} tagPolicy: sha256: {} artifacts: diff --git a/workflow/deploy/tensorflow-serving-base/src/Dockerfile b/workflow/deploy/tensorflow-serving-base/src/Dockerfile index 8c887b4..fe0168c 100644 --- a/workflow/deploy/tensorflow-serving-base/src/Dockerfile +++ b/workflow/deploy/tensorflow-serving-base/src/Dockerfile @@ -1,6 +1,11 @@ ARG FROM=max-one.local:5001/jetson/nano/ml-base FROM ${FROM} +MAINTAINER Helmut Hoffer von Ankershoffen + +# Inc on updates of base image +ENV BUMP=1 + # Number of parallel jobs when building TensorFlow serving ARG JOBS=1 @@ -8,11 +13,10 @@ ARG JOBS=1 ARG TF_CUDA_COMPUTE_CAPABILITIES ENV TF_CUDA_COMPUTE_CAPABILITIES ${TF_CUDA_COMPUTE_CAPABILITIES:-5.3,6.2,7.2} +# JETSON_MODEL used in .bazelrc ARG JETSON_MODEL ENV JETSON_MODEL ${JETSON_MODEL:-nano} -MAINTAINER Helmut Hoffer von Ankershoffen - # Install supervisord RUN apt-get update && \ apt-get install -y supervisor && \ @@ -105,4 +109,7 @@ EXPOSE 8500 # rest EXPOSE 8501 +RUN mkdir -p /meta && \ + date +"%FT%T%Z" > /meta/tensorflow-serving-base.build + CMD ["/usr/bin/supervisord"] diff --git a/workflow/deploy/tensorflow-serving/skaffold.yaml b/workflow/deploy/tensorflow-serving/skaffold.yaml index fe9d151..9473b9c 100644 --- a/workflow/deploy/tensorflow-serving/skaffold.yaml +++ b/workflow/deploy/tensorflow-serving/skaffold.yaml @@ -1,7 +1,6 @@ apiVersion: skaffold/v1beta11 kind: Config build: - local: {} tagPolicy: sha256: {} artifacts: diff --git a/workflow/deploy/tensorflow-serving/src/Dockerfile b/workflow/deploy/tensorflow-serving/src/Dockerfile index ec7ffc5..e2d59dd 100644 --- a/workflow/deploy/tensorflow-serving/src/Dockerfile +++ b/workflow/deploy/tensorflow-serving/src/Dockerfile @@ -3,6 +3,9 @@ FROM ${FROM} MAINTAINER Helmut Hoffer von Ankershoffen +# Inc on updates of base image +ENV BUMP=1 + # Install application server RUN pip install \ fastapi \ @@ -22,6 +25,7 @@ COPY /models/testdata/saved_model_half_plus_two_mkl /models/half_plus_two # Install testdata COPY /models/testdata /testdata +# Workdir is root WORKDIR / # Expose ports @@ -31,3 +35,6 @@ EXPOSE 80 EXPOSE 8500 # rest EXPOSE 8501 + +RUN mkdir -p /meta && \ + date +"%FT%T%Z" > /meta/tensorflow-serving.build diff --git a/workflow/deploy/tools/builder b/workflow/deploy/tools/builder index c0fd51e..1c4a338 100755 --- a/workflow/deploy/tools/builder +++ b/workflow/deploy/tools/builder @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Regular builder +# Regular remote builder ## Project to build project=$1 @@ -20,16 +20,15 @@ shift; shift; shift options="$@" cd $(dirname "$0")/../$project -echo "Building $IMAGES for project $project and Jetson model ${model} with FROM=from and $options ..." +echo "Building $IMAGES for project $project and Jetson model $model with FROM=$from and $options ..." ## Sync src from development machine to device -echo "Synching from development machine to ${model} for project $project ..." +echo "Synching from development machine to build@${model}-one.local for project $project ..." rsync -rlptza --delete -P src/ build@${model}-one.local:~/$project ## Build Docker image on device ssh build@${model}-one.local << EOF - echo "Building Docker image on ${model} for project $project ..." - if [ -z "$2" ]; then + if [ -z "$from" ]; then echo "Executing on build@${model}-one.local: docker build -t $project ~/$project $options" docker build -t $project ~/$project $options else @@ -47,6 +46,8 @@ do echo "Pushing $image for project $project..." ssh build@${model}-one.local "docker push $image" fi + # echo "Pulling $image on host ..." + # docker pull $image done echo "Building $IMAGES for project $project done." diff --git a/workflow/deploy/tools/builderx b/workflow/deploy/tools/builderx index 48abc5b..d793e17 100755 --- a/workflow/deploy/tools/builderx +++ b/workflow/deploy/tools/builderx @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Builder using buildx for bind-mounting during build +# Remote builder using buildx for bind-mounting during build ## Project to build project=$1 @@ -15,10 +15,10 @@ shift; shift options="$@" cd $(dirname "$0")/../$project -echo "Building $IMAGES for project $project and Jetson model ${model} ..." +echo "Building $IMAGES for project $project and Jetson model $model ..." ## Sync src from development machine to device -echo "Synching from development machine to ${model} for project $project ..." +echo "Synching from development machine to build@${model}-one.local for project $project ..." rsync -rlptza --delete --exclude=host -P src/ build@${model}-one.local:~/$project ## Sync directories on host into Docker build context for later bind-mounting during build @@ -72,4 +72,6 @@ do echo "Pushing $image for project $project..." ssh build@${model}-one.local "docker push $image" fi + # echo "Pulling $image on host ..." + # docker pull $image done diff --git a/workflow/deploy/tools/publish b/workflow/deploy/tools/publish index ff0aa2c..de6bbfc 100755 --- a/workflow/deploy/tools/publish +++ b/workflow/deploy/tools/publish @@ -2,15 +2,22 @@ ## Project to build project=$1 + +# Login at DockerHub user=$2 password=$3 -cd $(dirname "$0")/../$project -## Jetson model +# Jetson model if [[ -z "${JETSON_MODEL}" ]]; then - JETSON_MODEL="nano" + model="nano" +else + model=${JETSON_MODEL} fi -model=${4:-${JETSON_MODEL}} + +# Optional tag +tag=$4 + +cd $(dirname "$0")/../$project # Publish echo "Publish latest Docker image available found on ${model} to Docker Hub" @@ -20,4 +27,9 @@ ssh build@${model}-one.local << EOF echo "Publishing latest image of project $project to Docker Hub as $user/jetson-$model-$project:latest ..." docker tag $project $user/jetson-$model-$project:latest docker push $user/jetson-$model-$project:latest + if [[ ! -z "${tag}" ]]; then + echo "Additionally publish with as $user/jetson-$model-$project:$tag ..." + docker tag $project $user/jetson-$model-$project:$tag + docker push $user/jetson-$model-$project:$tag + fi EOF