github: workflow: add docker daemon to sdk build jobs

A from scratch docker daemon is required to build with the sdk containers, this is because the default docker in docker uses a abstracted docker socket that is hard to map the host folders to the container. Also the check is more complete when using a clean docker daemon. Signed-off-by: Matheus Castello <[email protected]>
torizon · Jan 14, 2025 · 8bb9391 · 8bb9391
1 parent d4fa1ac
commit 8bb9391
Show file tree

Hide file tree

Showing 6 changed files with 281 additions and 10 deletions.
diff --git a/.github/workflows/build-ccpp.yaml b/.github/workflows/build-ccpp.yaml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: torizonextras/torizon-dev:dev
-      options: --user root
+      options: --privileged --user root
     if: github.ref == 'refs/heads/bookworm' || github.ref == 'refs/heads/dev'
     name: C/C++ Container
     strategy:
@@ -27,18 +27,84 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Cleanup Space
+        run: |
+          # our builds are a bit larger than expected
+          # so we need to clean up the space from the runner
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf /usr/share/dotnet
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf /opt/ghc
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf "/usr/local/share/boost"
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo nsenter -t 1 -m -u -n -i -- apt purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
+            clang-6.0 lldb-6.0 lld-6.0 clang-format-6.0 clang-8 lldb-8 lld-8 clang-format-8 \
+            clang-9 lldb-9 lld-9 clangd-9 clang-format-9 dotnet-sdk-3.0 dotnet-sdk-3.1=3.1.101-1 \
+            esl-erlang firefox g++-8 g++-9 gfortran-8 gfortran-9 google-chrome-stable \
+            google-cloud-sdk ghc-8.0.2 ghc-8.2.2 ghc-8.4.4 ghc-8.6.2 ghc-8.6.3 ghc-8.6.4 \
+            ghc-8.6.5 ghc-8.8.1 ghc-8.8.2 ghc-8.8.3 ghc-8.10.1 cabal-install-2.0 cabal-install-2.2 \
+            cabal-install-2.4 cabal-install-3.0 cabal-install-3.2 heroku imagemagick \
+            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
+            mercurial apt-transport-https mono-complete mysql-client libmysqlclient-dev \
+            mysql-server mssql-tools unixodbc-dev yarn bazel chrpath libssl-dev libxft-dev \
+            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev php7.1 php7.1-bcmath \
+            php7.1-bz2 php7.1-cgi php7.1-cli php7.1-common php7.1-curl php7.1-dba php7.1-dev \
+            php7.1-enchant php7.1-fpm php7.1-gd php7.1-gmp php7.1-imap php7.1-interbase php7.1-intl \
+            php7.1-json php7.1-ldap php7.1-mbstring php7.1-mcrypt php7.1-mysql php7.1-odbc \
+            php7.1-opcache php7.1-pgsql php7.1-phpdbg php7.1-pspell php7.1-readline php7.1-recode \
+            php7.1-snmp php7.1-soap php7.1-sqlite3 php7.1-sybase php7.1-tidy php7.1-xml \
+            php7.1-xmlrpc php7.1-xsl php7.1-zip php7.2 php7.2-bcmath php7.2-bz2 php7.2-cgi \
+            php7.2-cli php7.2-common php7.2-curl php7.2-dba php7.2-dev php7.2-enchant php7.2-fpm \
+            php7.2-gd php7.2-gmp php7.2-imap php7.2-interbase php7.2-intl php7.2-json php7.2-ldap \
+            php7.2-mbstring php7.2-mysql php7.2-odbc php7.2-opcache php7.2-pgsql php7.2-phpdbg \
+            php7.2-pspell php7.2-readline php7.2-recode php7.2-snmp php7.2-soap php7.2-sqlite3 \
+            php7.2-sybase php7.2-tidy php7.2-xml php7.2-xmlrpc php7.2-xsl php7.2-zip php7.3 \
+            php7.3-bcmath php7.3-bz2 php7.3-cgi php7.3-cli php7.3-common php7.3-curl php7.3-dba \
+            php7.3-dev php7.3-enchant php7.3-fpm php7.3-gd php7.3-gmp php7.3-imap php7.3-interbase \
+            php7.3-intl php7.3-json php7.3-ldap php7.3-mbstring php7.3-mysql php7.3-odbc \
+            php7.3-opcache php7.3-pgsql php7.3-phpdbg php7.3-pspell php7.3-readline php7.3-recode \
+            php7.3-snmp php7.3-soap php7.3-sqlite3 php7.3-sybase php7.3-tidy php7.3-xml \
+            php7.3-xmlrpc php7.3-xsl php7.3-zip php7.4 php7.4-bcmath php7.4-bz2 php7.4-cgi \
+            php7.4-cli php7.4-common php7.4-curl php7.4-dba php7.4-dev php7.4-enchant php7.4-fpm \
+            php7.4-gd php7.4-gmp php7.4-imap php7.4-interbase php7.4-intl php7.4-json php7.4-ldap \
+            php7.4-mbstring php7.4-mysql php7.4-odbc php7.4-opcache php7.4-pgsql php7.4-phpdbg \
+            php7.4-pspell php7.4-readline php7.4-snmp php7.4-soap php7.4-sqlite3 php7.4-sybase \
+            php7.4-tidy php7.4-xml php7.4-xmlrpc php7.4-xsl php7.4-zip php-amqp php-apcu \
+            php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \
+            php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \
+            nodejs npm yarn python3 python3-pip python3-venv python3-dev python3-setuptools \
+            sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- apt autoremove -yq >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- apt-get autoclean -y >/dev/null 2>&1 || true
+
       - name: Get Host Absolute Workspace Path
         run: |
           echo "${{ github.workspace }}" >> abs-path
           cat abs-path
 
+      - name: Update torizon-utils module
+        run: |
+          chmod a+rwx . -R
+          sudo -u torizon pipx inject xonsh ./scripts/utils/pip/
+
+      - name: Run Docker Daemon
+        run: |
+          mkdir -p /var/lib/docker
+          dockerd \
+            --data-root=/var/lib/docker \
+            --host=unix:///var/run/docker-alt.sock \
+            --tls=false \
+            --host=tcp://0.0.0.0:2376 \
+            --userland-proxy-path=/usr/libexec/docker/docker-proxy \
+            --userland-proxy=false &
+          sleep 10
+
       - name: Build ${{ matrix.project.folder }} Dockerfile
         shell: xonsh {0}
         env:
           PROJECT_FOLDER: ${{ matrix.project.folder }}
           PROJECT_CONTAINER: ${{ matrix.project.container }}
           VENDOR: ${{ matrix.project.vendor }}
           DOCKER_LOGIN: localhost:5002
+          GITHUB_WORKSPACE_DOCKER: 2376
 
         run: |
           import os

diff --git a/.github/workflows/build-debug-ccpp.yaml b/.github/workflows/build-debug-ccpp.yaml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: torizonextras/torizon-dev:dev
-      options: --user root
+      options: --privileged --user root
     if: github.ref == 'refs/heads/bookworm' || github.ref == 'refs/heads/dev'
     name: C/C++ Container Debug
     strategy:
@@ -27,18 +27,85 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Cleanup Space
+        run: |
+          # our builds are a bit larger than expected
+          # so we need to clean up the space from the runner
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf /usr/share/dotnet
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf /opt/ghc
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf "/usr/local/share/boost"
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo nsenter -t 1 -m -u -n -i -- apt purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
+            clang-6.0 lldb-6.0 lld-6.0 clang-format-6.0 clang-8 lldb-8 lld-8 clang-format-8 \
+            clang-9 lldb-9 lld-9 clangd-9 clang-format-9 dotnet-sdk-3.0 dotnet-sdk-3.1=3.1.101-1 \
+            esl-erlang firefox g++-8 g++-9 gfortran-8 gfortran-9 google-chrome-stable \
+            google-cloud-sdk ghc-8.0.2 ghc-8.2.2 ghc-8.4.4 ghc-8.6.2 ghc-8.6.3 ghc-8.6.4 \
+            ghc-8.6.5 ghc-8.8.1 ghc-8.8.2 ghc-8.8.3 ghc-8.10.1 cabal-install-2.0 cabal-install-2.2 \
+            cabal-install-2.4 cabal-install-3.0 cabal-install-3.2 heroku imagemagick \
+            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
+            mercurial apt-transport-https mono-complete mysql-client libmysqlclient-dev \
+            mysql-server mssql-tools unixodbc-dev yarn bazel chrpath libssl-dev libxft-dev \
+            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev php7.1 php7.1-bcmath \
+            php7.1-bz2 php7.1-cgi php7.1-cli php7.1-common php7.1-curl php7.1-dba php7.1-dev \
+            php7.1-enchant php7.1-fpm php7.1-gd php7.1-gmp php7.1-imap php7.1-interbase php7.1-intl \
+            php7.1-json php7.1-ldap php7.1-mbstring php7.1-mcrypt php7.1-mysql php7.1-odbc \
+            php7.1-opcache php7.1-pgsql php7.1-phpdbg php7.1-pspell php7.1-readline php7.1-recode \
+            php7.1-snmp php7.1-soap php7.1-sqlite3 php7.1-sybase php7.1-tidy php7.1-xml \
+            php7.1-xmlrpc php7.1-xsl php7.1-zip php7.2 php7.2-bcmath php7.2-bz2 php7.2-cgi \
+            php7.2-cli php7.2-common php7.2-curl php7.2-dba php7.2-dev php7.2-enchant php7.2-fpm \
+            php7.2-gd php7.2-gmp php7.2-imap php7.2-interbase php7.2-intl php7.2-json php7.2-ldap \
+            php7.2-mbstring php7.2-mysql php7.2-odbc php7.2-opcache php7.2-pgsql php7.2-phpdbg \
+            php7.2-pspell php7.2-readline php7.2-recode php7.2-snmp php7.2-soap php7.2-sqlite3 \
+            php7.2-sybase php7.2-tidy php7.2-xml php7.2-xmlrpc php7.2-xsl php7.2-zip php7.3 \
+            php7.3-bcmath php7.3-bz2 php7.3-cgi php7.3-cli php7.3-common php7.3-curl php7.3-dba \
+            php7.3-dev php7.3-enchant php7.3-fpm php7.3-gd php7.3-gmp php7.3-imap php7.3-interbase \
+            php7.3-intl php7.3-json php7.3-ldap php7.3-mbstring php7.3-mysql php7.3-odbc \
+            php7.3-opcache php7.3-pgsql php7.3-phpdbg php7.3-pspell php7.3-readline php7.3-recode \
+            php7.3-snmp php7.3-soap php7.3-sqlite3 php7.3-sybase php7.3-tidy php7.3-xml \
+            php7.3-xmlrpc php7.3-xsl php7.3-zip php7.4 php7.4-bcmath php7.4-bz2 php7.4-cgi \
+            php7.4-cli php7.4-common php7.4-curl php7.4-dba php7.4-dev php7.4-enchant php7.4-fpm \
+            php7.4-gd php7.4-gmp php7.4-imap php7.4-interbase php7.4-intl php7.4-json php7.4-ldap \
+            php7.4-mbstring php7.4-mysql php7.4-odbc php7.4-opcache php7.4-pgsql php7.4-phpdbg \
+            php7.4-pspell php7.4-readline php7.4-snmp php7.4-soap php7.4-sqlite3 php7.4-sybase \
+            php7.4-tidy php7.4-xml php7.4-xmlrpc php7.4-xsl php7.4-zip php-amqp php-apcu \
+            php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \
+            php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \
+            nodejs npm yarn python3 python3-pip python3-venv python3-dev python3-setuptools \
+            sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- apt autoremove -yq >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- apt-get autoclean -y >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- df -h
+
       - name: Get Host Absolute Workspace Path
         run: |
           echo "${{ github.workspace }}" >> abs-path
           cat abs-path
 
+      - name: Update torizon-utils module
+        run: |
+          chmod a+rwx . -R
+          sudo -u torizon pipx inject xonsh ./scripts/utils/pip/
+
+      - name: Run Docker Daemon
+        run: |
+          mkdir -p /var/lib/docker
+          dockerd \
+            --data-root=/var/lib/docker \
+            --host=unix:///var/run/docker-alt.sock \
+            --tls=false \
+            --host=tcp://0.0.0.0:2376 \
+            --userland-proxy-path=/usr/libexec/docker/docker-proxy \
+            --userland-proxy=false &
+          sleep 10
+
       - name: Build ${{ matrix.project.folder }} Dockerfile
         shell: xonsh {0}
         env:
           PROJECT_FOLDER: ${{ matrix.project.folder }}
           PROJECT_CONTAINER: ${{ matrix.project.container }}
           VENDOR: ${{ matrix.project.vendor }}
           DOCKER_LOGIN: localhost:5002
+          GITHUB_WORKSPACE_DOCKER: 2376
 
         run: |
           import os
@@ -70,9 +137,7 @@ jobs:
           with open(".vscode/settings.json", "w") as file:
             json.dump(settings_json, file, indent=4)
 
-          # FIXME:  this is bad, but it's the only to make it work here
-          chmod 777 . -R
-
+          chmod a+rwx . -R
           xonsh ./.vscode/tasks.xsh run run-torizon-binfmt
           xonsh ./.vscode/tasks.xsh run @(f"build-debug-{os.environ['PROJECT_ARCH']}")
           xonsh ./.vscode/tasks.xsh run @(f"build-container-torizon-debug-{os.environ['PROJECT_ARCH']}")
diff --git a/.github/workflows/build-debug-python.yaml b/.github/workflows/build-debug-python.yaml
@@ -28,6 +28,11 @@ jobs:
           echo "${{ github.workspace }}" >> abs-path
           cat abs-path
 
+      - name: Update torizon-utils module
+        run: |
+          chmod a+rwx . -R
+          sudo -u torizon pipx inject xonsh ./scripts/utils/pip/
+
       - name: Build ${{ matrix.project.folder }} Dockerfile
         shell: xonsh {0}
         env:

diff --git a/.github/workflows/build-debug-rust.yaml b/.github/workflows/build-debug-rust.yaml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: torizonextras/torizon-dev:dev
-      options: --user root
+      options: --privileged --user root
     if: github.ref == 'refs/heads/bookworm' || github.ref == 'refs/heads/dev'
     name: Rust Container Debug
     strategy:
@@ -23,18 +23,84 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Cleanup Space
+        run: |
+          # our builds are a bit larger than expected
+          # so we need to clean up the space from the runner
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf /usr/share/dotnet
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf /opt/ghc
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf "/usr/local/share/boost"
+          sudo nsenter -t 1 -m -u -n -i -- rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo nsenter -t 1 -m -u -n -i -- apt purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
+            clang-6.0 lldb-6.0 lld-6.0 clang-format-6.0 clang-8 lldb-8 lld-8 clang-format-8 \
+            clang-9 lldb-9 lld-9 clangd-9 clang-format-9 dotnet-sdk-3.0 dotnet-sdk-3.1=3.1.101-1 \
+            esl-erlang firefox g++-8 g++-9 gfortran-8 gfortran-9 google-chrome-stable \
+            google-cloud-sdk ghc-8.0.2 ghc-8.2.2 ghc-8.4.4 ghc-8.6.2 ghc-8.6.3 ghc-8.6.4 \
+            ghc-8.6.5 ghc-8.8.1 ghc-8.8.2 ghc-8.8.3 ghc-8.10.1 cabal-install-2.0 cabal-install-2.2 \
+            cabal-install-2.4 cabal-install-3.0 cabal-install-3.2 heroku imagemagick \
+            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
+            mercurial apt-transport-https mono-complete mysql-client libmysqlclient-dev \
+            mysql-server mssql-tools unixodbc-dev yarn bazel chrpath libssl-dev libxft-dev \
+            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev php7.1 php7.1-bcmath \
+            php7.1-bz2 php7.1-cgi php7.1-cli php7.1-common php7.1-curl php7.1-dba php7.1-dev \
+            php7.1-enchant php7.1-fpm php7.1-gd php7.1-gmp php7.1-imap php7.1-interbase php7.1-intl \
+            php7.1-json php7.1-ldap php7.1-mbstring php7.1-mcrypt php7.1-mysql php7.1-odbc \
+            php7.1-opcache php7.1-pgsql php7.1-phpdbg php7.1-pspell php7.1-readline php7.1-recode \
+            php7.1-snmp php7.1-soap php7.1-sqlite3 php7.1-sybase php7.1-tidy php7.1-xml \
+            php7.1-xmlrpc php7.1-xsl php7.1-zip php7.2 php7.2-bcmath php7.2-bz2 php7.2-cgi \
+            php7.2-cli php7.2-common php7.2-curl php7.2-dba php7.2-dev php7.2-enchant php7.2-fpm \
+            php7.2-gd php7.2-gmp php7.2-imap php7.2-interbase php7.2-intl php7.2-json php7.2-ldap \
+            php7.2-mbstring php7.2-mysql php7.2-odbc php7.2-opcache php7.2-pgsql php7.2-phpdbg \
+            php7.2-pspell php7.2-readline php7.2-recode php7.2-snmp php7.2-soap php7.2-sqlite3 \
+            php7.2-sybase php7.2-tidy php7.2-xml php7.2-xmlrpc php7.2-xsl php7.2-zip php7.3 \
+            php7.3-bcmath php7.3-bz2 php7.3-cgi php7.3-cli php7.3-common php7.3-curl php7.3-dba \
+            php7.3-dev php7.3-enchant php7.3-fpm php7.3-gd php7.3-gmp php7.3-imap php7.3-interbase \
+            php7.3-intl php7.3-json php7.3-ldap php7.3-mbstring php7.3-mysql php7.3-odbc \
+            php7.3-opcache php7.3-pgsql php7.3-phpdbg php7.3-pspell php7.3-readline php7.3-recode \
+            php7.3-snmp php7.3-soap php7.3-sqlite3 php7.3-sybase php7.3-tidy php7.3-xml \
+            php7.3-xmlrpc php7.3-xsl php7.3-zip php7.4 php7.4-bcmath php7.4-bz2 php7.4-cgi \
+            php7.4-cli php7.4-common php7.4-curl php7.4-dba php7.4-dev php7.4-enchant php7.4-fpm \
+            php7.4-gd php7.4-gmp php7.4-imap php7.4-interbase php7.4-intl php7.4-json php7.4-ldap \
+            php7.4-mbstring php7.4-mysql php7.4-odbc php7.4-opcache php7.4-pgsql php7.4-phpdbg \
+            php7.4-pspell php7.4-readline php7.4-snmp php7.4-soap php7.4-sqlite3 php7.4-sybase \
+            php7.4-tidy php7.4-xml php7.4-xmlrpc php7.4-xsl php7.4-zip php-amqp php-apcu \
+            php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \
+            php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \
+            nodejs npm yarn python3 python3-pip python3-venv python3-dev python3-setuptools \
+            sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- apt autoremove -yq >/dev/null 2>&1 || true
+          sudo nsenter -t 1 -m -u -n -i -- apt-get autoclean -y >/dev/null 2>&1 || true
+
       - name: Get Host Absolute Workspace Path
         run: |
           echo "${{ github.workspace }}" >> abs-path
           cat abs-path
 
+      - name: Update torizon-utils module
+        run: |
+          chmod a+rwx . -R
+          sudo -u torizon pipx inject xonsh ./scripts/utils/pip/
+
+      - name: Run Docker Daemon
+        run: |
+          mkdir -p /var/lib/docker
+          dockerd \
+            --data-root=/var/lib/docker \
+            --host=unix:///var/run/docker-alt.sock \
+            --tls=false \
+            --host=tcp://0.0.0.0:2376 \
+            --userland-proxy-path=/usr/libexec/docker/docker-proxy \
+            --userland-proxy=false &
+          sleep 10
+
       - name: Build ${{ matrix.project.folder }} Dockerfile
         shell: xonsh {0}
         env:
           PROJECT_FOLDER: ${{ matrix.project.folder }}
           PROJECT_CONTAINER: ${{ matrix.project.container }}
           VENDOR: ${{ matrix.project.vendor }}
           DOCKER_LOGIN: localhost:5002
+          GITHUB_WORKSPACE_DOCKER: 2376
 
         run: |
           import os
@@ -66,9 +132,7 @@ jobs:
           with open(".vscode/settings.json", "w") as file:
             json.dump(settings_json, file, indent=4)
 
-          # FIXME:  this is bad, but it's the only to make it work here
-          chmod 777 . -R
-
+          chmod a+rwx . -R
           xonsh ./.vscode/tasks.xsh run run-torizon-binfmt
           xonsh ./.vscode/tasks.xsh run @(f"build-debug-{os.environ['PROJECT_ARCH']}")
           xonsh ./.vscode/tasks.xsh run @(f"copy-debug-executable-{os.environ['PROJECT_ARCH']}")

diff --git a/.github/workflows/build-python.yaml b/.github/workflows/build-python.yaml
@@ -28,6 +28,11 @@ jobs:
           echo "${{ github.workspace }}" >> abs-path
           cat abs-path
 
+      - name: Update torizon-utils module
+        run: |
+          chmod a+rwx . -R
+          sudo -u torizon pipx inject xonsh ./scripts/utils/pip/
+
       - name: Build ${{ matrix.project.folder }} Dockerfile
         shell: xonsh {0}
         env: