fixup! Some minor docker improvements

Signed-off-by: g2flyer <[email protected]>
g2flyer · Jan 20, 2024 · a55afd5 · a55afd5
1 parent 155883c
commit a55afd5
Show file tree

Hide file tree

Showing 41 changed files with 112 additions and 114 deletions.
diff --git a/bin/lib/common.sh b/bin/lib/common.sh
@@ -82,10 +82,13 @@ function force_to_ip()
 # -----------------------------------------
 # Check the environment for completeness
 # -----------------------------------------
-: "${PDO_HOME:-$(die Missing environment variable PDO_HOME)}"
-: "${PDO_HOSTNAME:-$(die Missing environment variable PDO_HOSTNAME)}"
-: "${PDO_INTERPRETER:-$(die Missing environment variable PDO_INTERPRETER)}"
-: "${PDO_LEDGER_KEY_ROOT:-$(die Missing environment variable PDO_LEDGER_KEY_ROOT)}"
-: "${PDO_LEDGER_TYPE:-$(die Missing environment variable PDO_LEDGER_TYPE)}"
-: "${PDO_LEDGER_URL:-$(die Missing environment variable PDO_LEDGER_URL)}"
-: "${PDO_SOURCE_ROOT:-$(die Missing environment variable PDO_SOURCE_ROOT)}"
+function check_pdo_env()
+{
+    : "${PDO_HOME:-$(die Missing environment variable PDO_HOME)}"
+    : "${PDO_HOSTNAME:-$(die Missing environment variable PDO_HOSTNAME)}"
+    : "${PDO_INTERPRETER:-$(die Missing environment variable PDO_INTERPRETER)}"
+    : "${PDO_LEDGER_KEY_ROOT:-$(die Missing environment variable PDO_LEDGER_KEY_ROOT)}"
+    : "${PDO_LEDGER_TYPE:-$(die Missing environment variable PDO_LEDGER_TYPE)}"
+    : "${PDO_LEDGER_URL:-$(die Missing environment variable PDO_LEDGER_URL)}"
+    : "${PDO_SOURCE_ROOT:-$(die Missing environment variable PDO_SOURCE_ROOT)}"
+}
diff --git a/build/__tools__/build.sh b/build/__tools__/build.sh
@@ -20,6 +20,7 @@ SCRIPTDIR="$(dirname $(readlink --canonicalize ${BASH_SOURCE}))"
 SRCDIR="$(realpath ${SCRIPTDIR}/../..)"
 
 source ${SRCDIR}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 # Automatically determine how many cores the host system has

diff --git a/build/__tools__/clean.sh b/build/__tools__/clean.sh
@@ -18,6 +18,7 @@ SCRIPTDIR="$(dirname $(readlink --canonicalize ${BASH_SOURCE}))"
 SRCDIR="$(realpath ${SCRIPTDIR}/../..)"
 
 source ${SRCDIR}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 # -----------------------------------------------------------------

diff --git a/build/__tools__/run-benchmarks.sh b/build/__tools__/run-benchmarks.sh
@@ -20,6 +20,7 @@ SCRIPTDIR="$(dirname $(readlink --canonicalize ${BASH_SOURCE}))"
 SRCDIR="$(realpath ${SCRIPTDIR}/../..)"
 
 source ${SRCDIR}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 # -----------------------------------------------------------------

diff --git a/build/__tools__/run-perf-tests.sh b/build/__tools__/run-perf-tests.sh
@@ -20,6 +20,7 @@ SCRIPTDIR="$(dirname $(readlink --canonicalize ${BASH_SOURCE}))"
 SRCDIR="$(realpath ${SCRIPTDIR}/../..)"
 
 source ${SRCDIR}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 # -----------------------------------------------------------------

diff --git a/build/__tools__/run-tests.sh b/build/__tools__/run-tests.sh
@@ -17,6 +17,7 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 PDO_LOG_LEVEL=${PDO_LOG_LEVEL:-info}

diff --git a/build/__tools__/verify-pre-build.sh b/build/__tools__/verify-pre-build.sh
@@ -25,6 +25,7 @@ SCRIPTDIR="$(dirname $(readlink --canonicalize ${BASH_SOURCE}))"
 SRCDIR="$(realpath ${SCRIPTDIR}/../..)"
 
 source ${SRCDIR}/bin/lib/common.sh
+check_pdo_env
 
 function warn () {
     recho "WARNING: $*"

diff --git a/build/__tools__/verify-pre-conf.sh b/build/__tools__/verify-pre-conf.sh
@@ -25,6 +25,7 @@ SCRIPTDIR="$(dirname $(readlink --canonicalize ${BASH_SOURCE}))"
 SRCDIR="$(realpath ${SCRIPTDIR}/../..)"
 
 source ${SRCDIR}/bin/lib/common.sh
+check_pdo_env
 
 function warn () {
     recho "WARNING: $*"

diff --git a/build/tests/multi-user.sh b/build/tests/multi-user.sh
@@ -17,6 +17,7 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 # -----------------------------------------------------------------

diff --git a/build/tests/service-test.sh b/build/tests/service-test.sh
@@ -17,6 +17,7 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 # -----------------------------------------------------------------

diff --git a/build/tests/unit-test.sh b/build/tests/unit-test.sh
@@ -17,6 +17,7 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_env
 check_python_version
 
 PDO_LOG_LEVEL=${PDO_LOG_LEVEL:-info}

diff --git a/common/crypto/verify_ias_report/fetch_ias_certificates.sh b/common/crypto/verify_ias_report/fetch_ias_certificates.sh
@@ -23,6 +23,7 @@
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+check_pdo_env
 
 IAS_CERTIFICATE_URL=$1
 

diff --git a/docker/Makefile b/docker/Makefile
@@ -43,8 +43,6 @@ PDO_GROUP_UID ?= $(shell id -g)
 DOCKER_DIR ?= ${PDO_SOURCE_ROOT}/docker
 
 DOCKER_USERNAME = $(LOGNAME)
-DOCKER_BUILDARGS += --build-arg PDO_HOSTNAME=$(PDO_HOSTNAME)
-DOCKER_BUILDARGS += --build-arg PDO_LEDGER_URL=$(PDO_LEDGER_URL)
 DOCKER_BUILDARGS += --build-arg UID=$(PDO_USER_UID)
 DOCKER_BUILDARGS += --build-arg GID=$(PDO_GROUP_UID)
 DOCKER_ARGS = $(DOCKER_BUILDARGS)
@@ -60,36 +58,43 @@ all : $(addprefix build_,$(IMAGES))
 rebuild_% : repository
 	@ docker build $(DOCKER_ARGS) \
 		--build-arg REBUILD=$(TIMESTAMP) \
-		--build-arg PDO_VERSION=${PDO_VERSION} \
-		--tag pdo_$*:${PDO_VERSION} \
+		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--tag pdo_$*:$(PDO_VERSION) \
 		--file $(DOCKER_DIR)/pdo_$*.dockerfile .
 
 build_% : repository
 	@ docker build $(DOCKER_ARGS) \
-		--build-arg PDO_VERSION=${PDO_VERSION} \
-		--tag pdo_$*:${PDO_VERSION} \
+		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--tag pdo_$*:$(PDO_VERSION) \
 		--file $(DOCKER_DIR)/pdo_$*.dockerfile .
 
+# docker build dependencies
+build_client: build_base
+build_services: build_services_base
+build_services_base: build_base
+build_ccf: build_ccf_base
+
+
 clean_% :
-	@ docker rmi -f pdo_$*:${PDO_VERSION}
+	@ docker rmi -f pdo_$*:$(PDO_VERSION)
 
 run_ccf : build_ccf stop_ccf
-	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer --network host --name ccf_container -P -d pdo_ccf:${PDO_VERSION}
+	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer --network host --name ccf_container -P -d pdo_ccf:$(PDO_VERSION)
 
 run_ccf_dev : build_ccf
-	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer -v ${PDO_SOURCE_ROOT}/docker/tools/:/project/pdo/tools -v ${PDO_SOURCE_ROOT}/:/project/pdo/src --entrypoint=/bin/bash -it --network host --name ccf_container_dev -P pdo_ccf:${PDO_VERSION} -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
+	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer -v ${PDO_SOURCE_ROOT}/docker/tools/:/project/pdo/tools -v ${PDO_SOURCE_ROOT}/:/project/pdo/src --entrypoint=/bin/bash -it --network host --name ccf_container_dev -P pdo_ccf:$(PDO_VERSION) -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
 
 run_services : build_base build_services_base build_services stop_services
-	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer --network host --name services_container -P -d pdo_services:${PDO_VERSION}
+	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer --network host --name services_container -P -d pdo_services:$(PDO_VERSION)
 
 run_services_dev : build_base build_services_base build_services
-	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer  -v ${PDO_SOURCE_ROOT}/docker/tools/:/project/pdo/tools -v ${PDO_SOURCE_ROOT}/:/project/pdo/src --entrypoint=/bin/bash -it --network host --name services_container_dev -P pdo_services:${PDO_VERSION} -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
+	@ docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer  -v ${PDO_SOURCE_ROOT}/docker/tools/:/project/pdo/tools -v ${PDO_SOURCE_ROOT}/:/project/pdo/src --entrypoint=/bin/bash -it --network host --name services_container_dev -P pdo_services:$(PDO_VERSION) -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
 
 run_client : build_base build_client
-	@docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer -it --network host --name client_container pdo_client:${PDO_VERSION} -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
+	@docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer -it --network host --name client_container pdo_client:$(PDO_VERSION) -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
 
 run_client_dev : build_base build_client
-	@docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer  -v ${PDO_SOURCE_ROOT}/docker/tools/:/project/pdo/tools -v ${PDO_SOURCE_ROOT}/:/project/pdo/src --entrypoint=/bin/bash -it --network host --name client_container_dev pdo_client:${PDO_VERSION} -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
+	@docker run -v $(DOCKER_DIR)/xfer/:/project/pdo/xfer  -v ${PDO_SOURCE_ROOT}/docker/tools/:/project/pdo/tools -v ${PDO_SOURCE_ROOT}/:/project/pdo/src --entrypoint=/bin/bash -it --network host --name client_container_dev pdo_client:$(PDO_VERSION) -c "stty cols $$(tput cols) rows $$(tput lines) && bash"
 
 stop_all : stop_ccf stop_services stop_client stop_ccf_dev stop_services_dev stop_client_dev
 
@@ -136,14 +141,14 @@ TEST_FILES += -f services_base.yaml
 TEST_FILES += -f ccf_base.yaml
 TEST_FILES += -f test.yaml
 
-build_test : repository build_base build_services_base build_ccf_base
-	PDO_VERSION=${PDO_VERSION} PDO_USER_UID=$(PDO_USER_UID) PDO_GROUP_UID=$(PDO_GROUP_UID) \
-		docker-compose $(TEST_FILES) build \
-			--build-arg PDO_VERSION=${PDO_VERSION}
+build_test : repository build_services build_ccf build_client
+
+test : clean_config clean_repository test_no_reset
+
+test_no_reset : build_test stop_all
+	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) up --abort-on-container-exit
+	PDO_VERSION=$(PDO_VERSION) docker-compose $(TEST_FILES) down
 
-test : clean_config clean_repository build_test
-	PDO_VERSION=${PDO_VERSION} docker-compose $(TEST_FILES) up --abort-on-container-exit
-	PDO_VERSION=${PDO_VERSION} docker-compose $(TEST_FILES) down
 
 # -----------------------------------------------------------------
 # Cleaning is a bit interesting because the containers don't go away

diff --git a/docker/base.yaml b/docker/base.yaml
@@ -18,9 +18,6 @@ version: "3.4"
 services:
   base_container:
     image: pdo_base:${PDO_VERSION:-latest}
-    build:
-      context: .
-      dockerfile: pdo_base.dockerfile
-      args:
-        ADD_APT_PKGS: ${ADD_APT_PKGS}
-        PDO_VERSION: ${PDO_VERSION:-latest}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
+    # double-build
diff --git a/docker/ccf_base.yaml b/docker/ccf_base.yaml
@@ -17,15 +17,9 @@ version: "3.4"
 services:
   ccf_container:
     image: pdo_ccf:${PDO_VERSION:-latest}
-    build:
-      context: .
-      dockerfile: pdo_ccf.dockerfile
-      args:
-        PDO_INTERPRETER: ${PDO_INTERPRETER:-wawaka}
-        PDO_LEDGER_TYPE: ccf
-        PDO_HOSTNAME: ${PDO_HOSTNAME}
-        PDO_LEDGER_URL: ${PDO_LEDGER_URL}
-        SGX_MODE: ${SGX_MODE:-SIM}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
+    # double-build
     container_name: ccf_container
     network_mode: "host"
     volumes:

diff --git a/docker/client_base.yaml b/docker/client_base.yaml
@@ -17,18 +17,8 @@ version: "3.4"
 services:
   client_container:
     image: pdo_client:${PDO_VERSION:-latest}
-    build:
-      context: .
-      dockerfile: pdo_client.dockerfile
-      args:
-        UID: ${PDO_USER_UID:-1000}
-        GID: ${PDO_GROUP_UID:-1000}
-        PDO_INTERPRETER: ${PDO_INTERPRETER:-wawaka}
-        PDO_LEDGER_TYPE: ccf
-        PDO_HOSTNAME: ${PDO_HOSTNAME}
-        PDO_LEDGER_URL: ${PDO_LEDGER_URL}
-        PDO_LOG_LEVEL: ${PDO_LOG_LEVEL:-info}
-        PDO_DEBUG_BUILD: ${PDO_DEBUG_BUILD:-0}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
     depends_on:
       - ccf_container
       - services_container

diff --git a/docker/pdo_ccf.dockerfile b/docker/pdo_ccf.dockerfile
@@ -20,9 +20,9 @@ FROM pdo_ccf_base:${PDO_VERSION}
 # -----------------------------------------------------------------
 # set up the PDO sources
 # -----------------------------------------------------------------
-ARG REBUILD 0
+ARG REBUILD=0
 
-ARG SGX_MODE SIM
+ARG SGX_MODE=SIM
 ENV SGX_MODE=$SGX_MODE
 
 ARG PDO_DEBUG_BUILD=0
@@ -49,11 +49,8 @@ RUN /project/pdo/tools/build_ccf.sh
 # Network ports for running services
 EXPOSE 6600
 
-ARG PDO_HOSTNAME
-ENV PDO_HOSTNAME=$PDO_HOSTNAME
-
-ARG PDO_LEDGER_URL
-ENV PDO_LEDGER_URL=$PDO_LEDGER_URL
+ARG PDO_HOSTNAME=localhost
+ENV PDO_HOSTNAME=${PDO_HOSTNAME}
 
 # Note that the entry point when specified with exec syntax
 # can be extended through the docker run interface far more

diff --git a/docker/pdo_ccf_base.dockerfile b/docker/pdo_ccf_base.dockerfile
@@ -39,7 +39,8 @@ RUN apt-get update \
         python3-virtualenv \
         virtualenv \
         net-tools \
-        wget
+        wget \
+        ${ADD_APT_PKGS}
 
 RUN echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu ${UBUNTU_NAME} main" >> /etc/apt/sources.list
 RUN curl https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add -

diff --git a/docker/pdo_client.dockerfile b/docker/pdo_client.dockerfile
@@ -35,10 +35,7 @@ USER $UNAME
 # -----------------------------------------------------------------
 # set up the PDO sources
 # -----------------------------------------------------------------
-ARG REBUILD 0
-
-ARG PDO_SOURCE_ROOT=/project/pdo/src
-ENV PDO_SOURCE_ROOT=${PDO_SOURCE_ROOT}
+ARG REBUILD=0
 
 ARG PDO_DEBUG_BUILD=0
 ENV PDO_DEBUG_BUILD=${PDO_DEBUG_BUILD}
@@ -57,7 +54,7 @@ ENV PDO_LOG_LEVEL=${PDO_LOG_LEVEL}
 
 # copy the source files into the image
 WORKDIR /project/pdo
-COPY --chown=${UNAME}:${UNAME} repository ${PDO_SOURCE_ROOT}
+COPY --chown=${UNAME}:${UNAME} repository /project/pdo/src
 
 # copy the tools because we want to be able to
 # use them even without a mount point after the
@@ -68,16 +65,13 @@ COPY --chown=${UNAME}:${UNAME} tools/*.sh ./
 # build it!!!
 RUN /project/pdo/tools/build_client.sh
 
-ARG PDO_HOSTNAME
+ARG PDO_HOSTNAME=localhost
 ENV PDO_HOSTNAME=${PDO_HOSTNAME}
 
-ARG PDO_LEDGER_URL
-ENV PDO_LEDGER_URL=${PDO_LEDGER_URL}
-
-ARG PDO_LEDGER_KEY_ROOT=/project/pdo/xfer/ccf/keys/
-ENV PDO_LEDGER_KEY_ROOT=${PDO_LEDGER_KEY_ROOT}
-
-# the client is set up for interactive access; the environment can be
-# set up by source /project/pdo/tools/start_client.sh with the arguments
-# to build a new client environment or copy one from the xfer directory
+RUN \
+  echo 'echo "\
+NOTE: to setup environment, \"source /project/pdo/tools/start_client.sh\"\n\
+   with the arguments \"--mode build\" to build a new client environment (default mode)\n\
+   or \"--mode copy\" to copy one from the xfer directory"\
+' > /project/pdo/.bashrc
 ENTRYPOINT [ "/bin/bash" ]
diff --git a/docker/pdo_services.dockerfile b/docker/pdo_services.dockerfile
@@ -20,7 +20,7 @@ FROM pdo_services_base:${PDO_VERSION}
 # -----------------------------------------------------------------
 # set up the PDO sources
 # -----------------------------------------------------------------
-ARG REBUILD 0
+ARG REBUILD=0
 
 ARG PDO_DEBUG_BUILD=0
 ENV PDO_DEBUG_BUILD=${PDO_DEBUG_BUILD}
@@ -55,12 +55,9 @@ EXPOSE 7001 7002 7003 7004 7005
 EXPOSE 7101 7102 7103 7104 7105
 EXPOSE 7201 7202 7203 7204 7205
 
-ARG PDO_HOSTNAME
+ARG PDO_HOSTNAME=localhost
 ENV PDO_HOSTNAME=$PDO_HOSTNAME
 
-ARG PDO_LEDGER_URL
-ENV PDO_LEDGER_URL=$PDO_LEDGER_URL
-
 # Note that the entry point when specified with exec syntax
 # can be extended through the docker run interface far more
 # easily than if you use the other specification format of

diff --git a/docker/services_base.yaml b/docker/services_base.yaml
@@ -18,17 +18,8 @@ version: "3.4"
 services:
   services_container:
     image: pdo_services:${PDO_VERSION:-latest}
-    build:
-      context: .
-      dockerfile: pdo_services.dockerfile
-      args:
-        PDO_INTERPRETER: ${PDO_INTERPRETER:-wawaka}
-        PDO_HOSTNAME: ${PDO_HOSTNAME}
-        PDO_LEDGER_TYPE: ccf
-        PDO_LEDGER_URL: ${PDO_LEDGER_URL}
-        SGX_MODE: ${SGX_MODE:-SIM}
-        PDO_LOG_LEVEL: ${PDO_LOG_LEVEL:-info}
-        PDO_DEBUG_BUILD: ${PDO_DEBUG_BUILD:-0}
+    # build containers only via Makefile and directly via docker build
+    # as we use containers also outside of docker-compose to prevent
     container_name: services_container
     network_mode: "host"
     depends_on: