Merge pull request #803 from KrisThielemans/docker_clean_build_files

[docker] reduce image size

.github/workflows/docker_build.yml

@@ -109,21 +109,20 @@ jobs:
         # Builds docker image from Docker file.
         shell: bash -l {0}
         run: |
+          set -ex;
           compose_command="docker-compose -f docker/docker-compose.yml" 
           if [[ ${{ matrix.tag }} == 'devel' ]]; then compose_command="$compose_command -f docker/docker-compose.devel.yml"; fi
           # $compose_command pull core
           $compose_command build --pull sirf
           echo "compose_command=$compose_command" >> $GITHUB_ENV
-
-      - name: Run docker container with tests
-        shell: bash -l {0}
-        run: |
-          ${{ env.compose_command }} run --rm -u jovyan --entrypoint /bin/bash sirf --login -c /devel/test.sh 1
           
       - name : docker save image
         # https://docs.docker.com/engine/reference/commandline/save/
         shell: bash -l {0}
         run: |
+          echo "info on layers and their sizes from docker history:"
+          sirf_image=`docker image ls | grep ${{ matrix.tag }} | awk '/sirf/ { print $3}'`
+          docker history "$sirf_image"
           docker save  synerbi/sirf:${{ matrix.tag }} | gzip > ${{ matrix.tag }}.tar.gz
           
       - name: Upload artifact of image.
@@ -157,6 +156,7 @@ jobs:
           # Builds docker image from Docker file.
           shell: bash -l {0}
           run: |
+            set -ex;
             # docker-compose -f docker/docker-compose.yml -f docker/docker-compose.srv-gpu.yml pull core 
             docker-compose -f docker/docker-compose.yml -f docker/docker-compose.srv-gpu.yml build --pull sirf
             
@@ -226,13 +226,9 @@ jobs:
         # Builds docker image from Docker file.
         shell: bash -l {0}
         run: |
+          set -ex;
           # ${{ env.compose_command }} pull sirf
           ${{ env.compose_command }} build --pull sirf
-
-      - name: Run docker service container with tests
-        shell: bash -l {0}
-        run: |
-          ${{ env.compose_command }} run --rm -u jovyan --entrypoint /bin/bash sirf --login -c /devel/test.sh 1
           
       - name : docker save image
         # https://docs.docker.com/engine/reference/commandline/save/

CHANGES.md

@@ -1,6 +1,11 @@
 # ChangeLog
 
-## Next
+## vx.x.x
+- docker image updates
+   - introduce `REMOVE_BUILD_FILES` variable. If set to 1 (which is the default),
+     most build files will be removed in the docker image.
+   - introduce `RUN_CTESTS` variable. If set to 1 (which is the default),
+     the CTests will be run while building the image.
 - Build Gadgetron master. Requires new Ubuntu packages: libdcmtk-dev, libpugixml-dev, libgflags-dev, 
   libssl-dev, libcurl4-openssl-dev, pkg-config, golang, libboost-coroutine-dev, libboost-context-dev, libboost-random-dev. 
 - Added SuperBuild project dependency: range-v3, RocksDB, Date, mrd-storage-server

docker/DocForDevelopers.md

@@ -15,11 +15,18 @@ export NUM_PARALLEL_BUILDS=9
 ```
 This will be passed to `cmake --build -j`.
 
+By default, most build files will be removed while building the docker image to save some space. If you are a developer, you can set
+the environment variable `REMOVE_BUILD_FILES` to 0 before building the image to prevent this.
+
+By default, the CTests are normally run while building the docker image. Note that this takes some time. You can switch this off by setting the environment variable `RUN_CTEST=0` before building the image.
+Note however that if you have set `REMOVE_BUILD_FILES=1`, you cannot run the CTests on an existing image (unless you rebuild from source of course).
+
 Of course, if you are using `bash`, you can specify any of these for a specific run, e.g.
 ```
 NUM_PARALLEL_BUILDS=9 ./sirf-compose build sirf
 ```
 
+
 ## `ccache`
 
 `ccache` is used in the container to speed up rebuilding images from scratch.

docker/Dockerfile

@@ -128,6 +128,12 @@ COPY user_sirf-ubuntu.sh .
 # SIRF-SuperBuild version
 ARG SIRF_SB_URL="https://github.com/SyneRBI/SIRF-SuperBuild"
 ARG SIRF_SB_TAG="master"
+
+# set if we want to remove the build files
+ARG REMOVE_BUILD_FILES=1
+# set if we want to run the tests during build
+ARG RUN_CTEST=1
+
 # speeding up the cmake build
 # (implementation note: changing this in your environment will invalidate the docker cache sadly)
 ARG NUM_PARALLEL_BUILDS="2"
@@ -142,10 +148,11 @@ ARG BUILD_FLAGS="\
  -DUSE_NiftyPET=OFF\
  -DBUILD_siemens_to_ismrmrd=ON -DBUILD_pet_rd_tools=ON"
 ARG EXTRA_BUILD_FLAGS="-DBUILD_CIL=OFF"
-RUN bash user_sirf-ubuntu.sh && rm user_sirf-ubuntu.sh
 
-# give everyone write and execute permission
-RUN chmod -R go+rwX /opt/SIRF-SuperBuild/INSTALL
+# build it, run ctest if required, and give everyone write and execute permission
+RUN bash user_sirf-ubuntu.sh \
+    && rm user_sirf-ubuntu.sh \
+    && chmod -R go+rwX /opt/SIRF-SuperBuild/INSTALL
 
 ENV DEBIAN_FRONTEND ''
 

docker/README.md

@@ -122,7 +122,7 @@ A brief list of everything important to know for a basic working knowledge of do
 - *Base image*: the starting point for building a Docker image
     + analogous to a clean OS (in this case `ubuntu:20.04`)
 - *Layer*: a single build step
-    + usually represented by a single line in a `Dockerfile` (e.g. `apt-get install cmake`)
+    + usually represented by a single `RUN` or `COPY` line in a `Dockerfile` (e.g. `RUN apt-get install cmake`)
 - *Image*: a sequence of *layers* (applied on top of a *base image*)
     + analogous to a clean OS with `SIRF` installed (in this case *tagged* `synerbi/sirf`)
 - *Container*: a sandboxed workspace derived from an *image*
@@ -312,9 +312,10 @@ docker rmi <IMAGEID>
 
 ## Notes
 
-- Tests of SIRF and other installed packages can be run as follows:
+- Since SIRF 3.5, by default the build files are removed on the docker image. This can be changed by [setting an environment variable](./DocForDevelopers.md#Useful-environment-variables).
+If you have built the image while keeping all build files (or re-build them in an existing container), tests of SIRF and other installed packages can be run as follows:
 ```bash
-(py2) sirf:~$ sudo -Hu jovyan bash --login -c /devel/test.sh
+sudo -Hu jovyan bash --login -c /devel/test.sh
 ```
 - Currently all `compose` files call the container `sirf`. You could edit the `.yml` file if you
 want to run different versions.

docker/devel/test.sh

@@ -15,19 +15,17 @@ pip install -U -r "$this"/requirements-test.txt
 pushd $SIRF_PATH/../..
 
 # start gadgetron
-GCONFIG=./INSTALL/share/gadgetron/config/gadgetron.xml
-[ -f "$GCONFIG" ] || cp "$GCONFIG".example "$GCONFIG"
 [ -f ./INSTALL/bin/gadgetron ] && ./INSTALL/bin/gadgetron >& gadgetron.log&
 
-ctest -VV
+ctest --output-on-failure
+
 ret=$?
 [ -n "$(pidof gadgetron)" ] && kill -n 15 $(pidof gadgetron)
 
 # print for debugging
 [ "$DEBUG" != 0 ] && cat builds/SIRF/build/CMakeCache.txt
-[ "$DEBUG" != 0 ] && cat builds/SIRF/build/Testing/Temporary/LastTest.log
 # may exceed 4MB travis log limit
-[ "$DEBUG" != 0 ] && [ -f gadgetron.log ] && cat gadgetron.log
+[ "$DEBUG" != 0 ] && [ -f gadgetron.log ] && tail -n 500 gadgetron.log
 
 popd
 

docker/docker-compose.yml

@@ -15,6 +15,8 @@ services:
     SIRF_SB_URL: ${SIRF_SB_URL:-https://github.com/SyneRBI/SIRF-SuperBuild}
     SIRF_SB_TAG: ${SIRF_SB_TAG:-master}
     NUM_PARALLEL_BUILDS: ${NUM_PARALLEL_BUILDS:-2}
+    REMOVE_BUILD_FILES: ${REMOVE_BUILD_FILES:-1}
+    RUN_CTEST: ${RUN_CTEST:-1}
    cache_from:
     - synerbi/sirf:core
     - synerbi/sirf:latest

docker/user_sirf-ubuntu.sh

@@ -1,12 +1,21 @@
 #!/usr/bin/env bash
+# This script builds the SIRF-SuperBuild
+# It relies on certain env variables to be set and should not be used
+# outside of the "docker build" setting
+
 [ -f .bashrc ] && . .bashrc
 set -ev
+# set default installation location
 INSTALL_DIR="${1:-/opt}"
 # set default URL/tag
 : ${SIRF_SB_URL:=https://github.com/SyneRBI/SIRF-SuperBuild}
 : ${SIRF_SB_TAG:=master}
 # set default number of parallel builds
 : ${NUM_PARALLEL_BUILDS:=2}
+# set default for cleaning up
+: ${REMOVE_BUILD_FILES:=1}
+# set default for running ctest during the build
+: ${RUN_CTEST:=1}
 
 git clone "$SIRF_SB_URL" --recursive "$INSTALL_DIR"/SIRF-SuperBuild
 cd $INSTALL_DIR/SIRF-SuperBuild
@@ -20,5 +29,28 @@ echo "COMPILER_FLAGS: $COMPILER_FLAGS"
 echo "BUILD+EXTRA FLAGS: $BUILD_FLAGS $EXTRA_BUILD_FLAGS"
 
 cmake $BUILD_FLAGS $EXTRA_BUILD_FLAGS $COMPILER_FLAGS .
+# save installation location for below
+CMAKE_INSTALL_PREFIX="${INSTALL_DIR}/SIRF-SuperBuild/INSTALL"
 
 cmake --build . -j ${NUM_PARALLEL_BUILDS}
+
+if [ "$RUN_CTEST" = 1 ]; then
+    source "$CMAKE_INSTALL_PREFIX"/bin/env_sirf.sh
+    # start gadgetron
+    [ -f "$CMAKE_INSTALL_PREFIX"/bin/gadgetron ] && "$CMAKE_INSTALL_PREFIX"/bin/gadgetron >& gadgetron.log&
+    ctest --output-on-failure
+    [ -n "$(pidof gadgetron)" ] && kill -n 15 $(pidof gadgetron)
+fi
+
+if [ "$REMOVE_BUILD_FILES" = 1 ]; then
+    echo  "Removing most build files"
+    rm -rf builds gadgetron.log
+    for s in sources/*/.git; do
+        cd $s/..
+        git clean -fdx
+        cd ../..
+    done
+else
+    echo  "Keeping build files"
+fi
+