From a8529182f1e56817dc0331805bc8ebb36ce89d22 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 4 Feb 2025 10:31:50 +0100 Subject: [PATCH 01/11] Fixes: compiler segfault on ARM64 The CI crashes with: ``` 21 370.9 [ 72%] Building C object tests/CMakeFiles/messaging_tests.dir/messaging_tests.c.o 21 371.1 cc: internal compiler error: Segmentation fault signal terminated program cc1 21 371.1 Please submit a full bug report, with preprocessed source (by using -freport-bug). 21 371.1 See for instructions. ``` when trying to build c-green 1.6.2. --- .github/install-openvas-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/install-openvas-dependencies.sh b/.github/install-openvas-dependencies.sh index 1f80d8bf9..23670d2b1 100755 --- a/.github/install-openvas-dependencies.sh +++ b/.github/install-openvas-dependencies.sh @@ -35,7 +35,7 @@ apt-get update && apt-get install --no-install-recommends --no-install-suggests libhiredis-dev \ && rm -rf /var/lib/apt/lists/* -curl -L -o cgreen.tar.gz https://github.com/cgreen-devs/cgreen/archive/refs/tags/1.6.2.tar.gz -k -tar -xzf cgreen.tar.gz && cd cgreen-1.6.2 +curl -L -o cgreen.tar.gz https://github.com/cgreen-devs/cgreen/archive/refs/tags/1.6.3.tar.gz -k +tar -xzf cgreen.tar.gz && cd cgreen-1.6.3 make install ldconfig From 52a37cd94cb1dc02f4d49c73b93e985e7d803bad Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 4 Feb 2025 11:25:39 +0100 Subject: [PATCH 02/11] Removing impacket in oldstable for now It breaks the oldstable build when creating an image for arm64. Since it should not be used in production anyway and we just use it for testing if we can still compile I think it is ok to remove runtime depdencies for now at least until a solution is found. --- .docker/prod-oldstable.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/prod-oldstable.Dockerfile b/.docker/prod-oldstable.Dockerfile index d43da964e..92e9c47ce 100644 --- a/.docker/prod-oldstable.Dockerfile +++ b/.docker/prod-oldstable.Dockerfile @@ -61,7 +61,7 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge # nasl_pread: Failed to execute child process “netstat” (No such file or directory) net-tools \ # for openvas-smb support - python3-impacket \ + # python3-impacket \ libgnutls30 \ libgssapi3-heimdal \ libkrb5-26-heimdal \ From 572ee93f4194a4442dea254f156ef02c0353aed7 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 4 Feb 2025 11:55:34 +0100 Subject: [PATCH 03/11] Using github version in oldstable The delivered cgreen version within oldstable seems to run in a similiar issue then the 1.6.2 version used previously within debian:stable. For this reason we use the github version 1.6.3. --- .docker/prod-oldstable.Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.docker/prod-oldstable.Dockerfile b/.docker/prod-oldstable.Dockerfile index 92e9c47ce..7e6e5e643 100644 --- a/.docker/prod-oldstable.Dockerfile +++ b/.docker/prod-oldstable.Dockerfile @@ -34,11 +34,15 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge libpopt0 \ libcurl4-gnutls-dev \ libcurl4 \ - libcgreen1-dev \ + # libcgreen1-dev \ libhiredis-dev \ libkrb5-dev \ && rm -rf /var/lib/apt/lists/* +RUN curl -L -o cgreen.tar.gz https://github.com/cgreen-devs/cgreen/archive/refs/tags/1.6.3.tar.gz -k +RUN tar -xzf cgreen.tar.gz && cd cgreen-1.6.3 +RUN make install +RUN ldconfig COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DINSTALL_OLD_SYNC_SCRIPT=OFF -B/build /source RUN DESTDIR=/install cmake --build /build -- install From 8a702a41917b1592f12e3881696561a9f46bb755 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 5 Feb 2025 18:16:47 +0100 Subject: [PATCH 04/11] CI: move building the rust binary inside docker image To prevent issues when building arm64 and simplify the setup the building of our rust code is moved from external source into the docker image. This should make maintaing the images a bit easier in the cost of slower image build time. Additionally we get rid of the qemu setup in favor of a aarch64 based internal runner. Revert "Removing impacket in oldstable for now" This reverts commit 61b911d004ee75bdbd8829b68a426171d673573b. --- .docker/prod-oldstable.Dockerfile | 9 +-- .docker/prod-testing.Dockerfile | 17 +++-- .docker/prod.Dockerfile | 20 ++++-- .github/workflows/build.yml | 3 + .github/workflows/functional.yaml | 1 + .../workflows/push-container-oldstable.yml | 53 ++++++--------- .github/workflows/push-container-testing.yml | 54 ++++++--------- .github/workflows/push-container.yml | 68 +++++++------------ 8 files changed, 96 insertions(+), 129 deletions(-) diff --git a/.docker/prod-oldstable.Dockerfile b/.docker/prod-oldstable.Dockerfile index 7e6e5e643..470e6f5e1 100644 --- a/.docker/prod-oldstable.Dockerfile +++ b/.docker/prod-oldstable.Dockerfile @@ -46,9 +46,9 @@ RUN ldconfig COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DINSTALL_OLD_SYNC_SCRIPT=OFF -B/build /source RUN DESTDIR=/install cmake --build /build -- install +# TODO: add rust? FROM registry.community.greenbone.net/community/gvm-libs:${GVM_LIBS_VERSION} -ARG TARGETPLATFORM RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ bison \ libglib2.0-0 \ @@ -65,7 +65,7 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge # nasl_pread: Failed to execute child process “netstat” (No such file or directory) net-tools \ # for openvas-smb support - # python3-impacket \ + python3-impacket \ libgnutls30 \ libgssapi3-heimdal \ libkrb5-26-heimdal \ @@ -79,11 +79,6 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge zlib1g\ && rm -rf /var/lib/apt/lists/* COPY .docker/openvas.conf /etc/openvas/ -# must be pre built within the rust dir and moved to the bin dir -# usually this image is created within in a ci ensuring that the -# binary is available. -COPY assets/$TARGETPLATFORM/scannerctl /usr/local/bin/scannerctl -RUN chmod +x /usr/local/bin/scannerctl COPY --from=build /install/ / COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ COPY --from=openvas-smb /usr/local/bin/ /usr/local/bin/ diff --git a/.docker/prod-testing.Dockerfile b/.docker/prod-testing.Dockerfile index c3175e2ed..1f3a51273 100644 --- a/.docker/prod-testing.Dockerfile +++ b/.docker/prod-testing.Dockerfile @@ -4,6 +4,7 @@ ARG REPOSITORY=greenbone/openvas-scanner ARG GVM_LIBS_VERSION=testing-edge FROM greenbone/openvas-smb:testing-edge AS openvas-smb +FROM rust AS rust FROM registry.community.greenbone.net/community/gvm-libs:${GVM_LIBS_VERSION} AS build COPY . /source @@ -42,8 +43,17 @@ COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DINSTALL_OLD_SYNC_SCRIPT=OFF -B/build /source RUN DESTDIR=/install cmake --build /build -- install +COPY --from=rust /usr/local/cargo/ /usr/local/cargo/ +COPY --from=rust /usr/local/rustup/ /usr/local/rustup/ +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH +RUN apt update && apt install -y ca-certificates +RUN cargo build --release +RUN cp target/release/openvasd /install/usr/local/bin +RUN cp target/release/scannerctl /install/usr/local/bin + FROM registry.community.greenbone.net/community/gvm-libs:${GVM_LIBS_VERSION} -ARG TARGETPLATFORM RUN apt-get update RUN apt-get install --no-install-recommends --no-install-suggests -y \ bison \ @@ -76,11 +86,6 @@ RUN apt-get install --no-install-recommends --no-install-suggests -y \ zlib1g RUN rm -rf /var/lib/apt/lists/* COPY .docker/openvas.conf /etc/openvas/ -# must be pre built within the rust dir and moved to the bin dir -# usually this image is created within in a ci ensuring that the -# binary is available. -COPY assets/$TARGETPLATFORM/scannerctl /usr/local/bin/scannerctl -RUN chmod +x /usr/local/bin/scannerctl COPY --from=build /install/ / COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ COPY --from=openvas-smb /usr/local/bin/ /usr/local/bin/ diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile index 029a162a2..81b0d373a 100644 --- a/.docker/prod.Dockerfile +++ b/.docker/prod.Dockerfile @@ -2,6 +2,7 @@ ARG VERSION=edge # this allows to work on forked repository ARG REPOSITORY=greenbone/openvas-scanner +FROM rust AS rust FROM greenbone/openvas-smb AS openvas-smb FROM registry.community.greenbone.net/community/gvm-libs:${VERSION} AS build @@ -10,9 +11,20 @@ RUN sh /source/.github/install-openvas-dependencies.sh COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DINSTALL_OLD_SYNC_SCRIPT=OFF -B/build /source RUN DESTDIR=/install cmake --build /build -- install +WORKDIR /source/rust +COPY --from=rust /usr/local/cargo/ /usr/local/cargo/ +COPY --from=rust /usr/local/rustup/ /usr/local/rustup/ +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH +RUN apt update && apt install -y ca-certificates +RUN cargo build --release +RUN cp target/release/openvasd /install/usr/local/bin +RUN cp target/release/scannerctl /install/usr/local/bin +# Do we want to copy feed verifier as well? +# RUN cp release/feed-verifier /install/bin FROM registry.community.greenbone.net/community/gvm-libs:${VERSION} -ARG TARGETPLATFORM RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ bison \ libglib2.0-0 \ @@ -42,13 +54,11 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge libssh-4 \ && rm -rf /var/lib/apt/lists/* COPY .docker/openvas.conf /etc/openvas/ + + # must be pre built within the rust dir and moved to the bin dir # usually this image is created within in a ci ensuring that the # binary is available. -COPY assets/$TARGETPLATFORM/scannerctl /usr/local/bin/scannerctl -COPY assets/$TARGETPLATFORM/openvasd /usr/local/bin/openvasd -RUN chmod +x /usr/local/bin/scannerctl -RUN chmod +x /usr/local/bin/openvasd COPY --from=build /install/ / COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ COPY --from=openvas-smb /usr/local/bin/ /usr/local/bin/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c06118f2c..3927c278f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,9 @@ jobs: run: | cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer cmake --build build + # TODO: change that to cargo check to check if it would be buildable but + # move the actual when it is needed - mainly on release + # This could significantly improve the speed of the CI Rust: runs-on: "ubuntu-latest" steps: diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 46e67782f..3ee422a0d 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -45,6 +45,7 @@ jobs: run: | cmake -Bbuild -DCMAKE_BUILD_TYPE=Release cmake --build build -- install + # TODO: build instead a x86_64 version - uses: actions/download-artifact@v4 with: name: rs-binaries diff --git a/.github/workflows/push-container-oldstable.yml b/.github/workflows/push-container-oldstable.yml index dddb33621..c859f38d0 100644 --- a/.github/workflows/push-container-oldstable.yml +++ b/.github/workflows/push-container-oldstable.yml @@ -17,45 +17,30 @@ on: jobs: debian_oldstable: - name: debian:oldstable - runs-on: ubuntu-latest + name: ghcr:debian:oldstable + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Container build and push 3rd gen + id: build-and-push + uses: greenbone/actions/container-build-push-generic@v3 with: - name: rs-binaries - path: assets - - name: "Set labels and tags" - id: old_stable_meta - uses: docker/metadata-action@v5 - with: - images: ${{ github.repository }} - labels: | + build-docker-file: .docker/prod.Dockerfile + build-args: | + REPOSITORY=${{ github.repository }} + cosign-key: ${{ secrets.cosign_key_opensight }} + cosign-key-password: ${{ secrets.cosign_password_opensight }} + # The tlog function does not currently support an ed25519 key. + cosign-tlog-upload: "false" + image-labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs - flavor: latest=false # no auto latest container tag for git tags - tags: | - # for the images provided for debian:oldstable we just provide - # oldstable on an new version or oldstable-edge when it is on main. - # oldstable-branch-sha on a branch + image-tags: | type=raw,value=oldstable,enable=${{ inputs.is_latest_tag }} type=raw,value=oldstable-edge,enable=${{ github.ref_name == 'main' }} type=raw,value=oldstable-{{branch}}-{{sha}},enable=${{ github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' }} type=ref,event=pr - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - username: ${{ secrets.dockerhub_user }} - password: ${{ secrets.dockerhub_token }} - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: . - push: ${{ github.event_name != 'pull_request' && (github.ref_type == 'tag' || github.ref_name == 'main') }} - file: .docker/prod-oldstable.Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.old_stable_meta.outputs.tags }} - labels: ${{ steps.old_stable_meta.outputs.labels }} + registry: ${{ vars.IMAGE_REGISTRY }} + registry-username: ${{ github.actor }} + registry-password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/push-container-testing.yml b/.github/workflows/push-container-testing.yml index 80532c320..d17a463e7 100644 --- a/.github/workflows/push-container-testing.yml +++ b/.github/workflows/push-container-testing.yml @@ -16,46 +16,32 @@ on: required: true jobs: + # TODO: do we need to push or is building enough? debian_testing: - name: debian:testing - runs-on: ubuntu-latest + name: ghcr:debian:testing + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + - name: Checkout repository + uses: actions/checkout@v4 + - name: Container build and push 3rd gen + id: build-and-push + uses: greenbone/actions/container-build-push-generic@v3 with: - name: rs-binaries - path: assets - - name: "Set labels and tags" - id: test_meta - uses: docker/metadata-action@v5 - with: - images: ${{ github.repository }} - labels: | + build-docker-file: .docker/prod.Dockerfile + build-args: | + REPOSITORY=${{ github.repository }} + cosign-key: ${{ secrets.cosign_key_opensight }} + cosign-key-password: ${{ secrets.cosign_password_opensight }} + # The tlog function does not currently support an ed25519 key. + cosign-tlog-upload: "false" + image-labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs - flavor: latest=false # no auto latest container tag for git tags - tags: | - # for the images provided for debian:testing we just provide - # testing on an new version or testing-edge when it is on main. - # testing-branch-sha on a branch + image-tags: | type=raw,value=testing,enable=${{ inputs.is_latest_tag }} type=raw,value=testing-edge,enable=${{ github.ref_name == 'main' }} type=raw,value=testing-{{branch}}-{{sha}},enable=${{ github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' }} type=ref,event=pr - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - username: ${{ secrets.dockerhub_user }} - password: ${{ secrets.dockerhub_token }} - - name: Build and push Container image - uses: docker/build-push-action@v6 - with: - context: . - push: ${{ github.event_name != 'pull_request' && (github.ref_type == 'tag' || github.ref_name == 'main') }} - file: .docker/prod-testing.Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.test_meta.outputs.tags }} - labels: ${{ steps.test_meta.outputs.labels }} + registry: ${{ vars.IMAGE_REGISTRY }} + registry-username: ${{ github.actor }} + registry-password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/push-container.yml b/.github/workflows/push-container.yml index dbad106eb..45dfd08d9 100644 --- a/.github/workflows/push-container.yml +++ b/.github/workflows/push-container.yml @@ -41,26 +41,29 @@ permissions: pull-requests: write jobs: - debian_stable: - name: debian:stable - runs-on: ubuntu-latest + + debian_stable_arm64: + name: ghcr:debian:stable:arm64 + runs-on: self-hosted-generic-arm64 steps: - - name: Checkout + - name: Checkout repository uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: rs-binaries - path: assets - - name: "Set labels and tags" - id: meta - uses: docker/metadata-action@v5 + - name: Container build and push 3rd gen + id: build-and-push + uses: greenbone/actions/container-build-push-generic@v3 with: - images: ${{ github.repository }} - labels: | + image-platforms: linux/arm64 + build-docker-file: .docker/prod.Dockerfile + build-args: | + REPOSITORY=${{ github.repository }} + cosign-key: ${{ secrets.cosign_key_opensight }} + cosign-key-password: ${{ secrets.cosign_password_opensight }} + # The tlog function does not currently support an ed25519 key. + cosign-tlog-upload: "false" + image-labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs - flavor: latest=false # no auto latest container tag for git tags - tags: | + image-tags: | # when IS_LATEST_TAG is set create a stable and a latest tag type=raw,value=latest,enable=${{ inputs.is_latest_tag }} type=raw,value=stable,enable=${{ inputs.is_latest_tag }} @@ -73,37 +76,18 @@ jobs: type=raw,value={{branch}}-{{sha}},enable=${{ github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' }} # use pr-$PR_ID for pull requests (will not be uploaded) type=ref,event=pr - - name: Login to DockerHub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - username: ${{ secrets.dockerhub_user }} - password: ${{ secrets.dockerhub_token }} - - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: . - push: ${{ github.event_name != 'pull_request' && (github.ref_type == 'tag' || github.ref_name == 'main') }} - file: .docker/prod.Dockerfile - build-args: | - REPOSITORY=${{ github.repository }} - platforms: linux/amd64,linux/aarch64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - ghcr_debian_stable: + registry: ${{ vars.IMAGE_REGISTRY }} + registry-username: ${{ github.actor }} + registry-password: ${{ secrets.GITHUB_TOKEN }} + scout-user: ${{ secrets.dockerhub_user }} + scout-password: ${{ secrets.dockerhub_token }} + + debian_stable: name: ghcr:debian:stable runs-on: "ubuntu-latest" steps: - name: Checkout repository uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: rs-binaries - path: assets - name: Container build and push 3rd gen id: build-and-push uses: greenbone/actions/container-build-push-generic@v3 @@ -145,8 +129,6 @@ jobs: image-labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs - artifact-name: rs-binaries - artifact-path: assets service: openvas-scanner secrets: COSIGN_KEY_OPENSIGHT: ${{ secrets.cosign_key_opensight }} From d4d19b44e321337430fb294602ae6397896a4d07 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 09:57:25 +0100 Subject: [PATCH 05/11] Adjust feed image --- .github/prepare-feed.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/prepare-feed.sh b/.github/prepare-feed.sh index c64963687..9c1802eab 100644 --- a/.github/prepare-feed.sh +++ b/.github/prepare-feed.sh @@ -4,7 +4,7 @@ # workflow. [ -z $FEED_DIR ] && FEED_DIR="/var/lib/openvas/plugins" DOCKER_CMD=docker -FEED_IMAGE="greenbone/vulnerability-tests" +FEED_IMAGE="registry.community.greenbone.net/community/vulnerability-tests" set -e printf "Copying feed $FEED_IMAGE " FEED_VERSION=$($DOCKER_CMD run --rm $FEED_IMAGE sh -c 'ls /var/lib/openvas/' | sort -r | head -n 1) From 53217b02f05a6321f487f2f07e3e182fad46a729 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 10:49:25 +0100 Subject: [PATCH 06/11] compile release on when necessary Instead of building the binaries everytime it is build when they are needed. This speeds up the regular CI checks significantly in the costs of slowing down functional tests and release. As we release less often then building and usually just want to know if the change compiles it is replaced with `cargo check`. --- .github/actions/compile-aarch64/action.yml | 38 +++++++++++++++++++++ .github/actions/compile-x86_64/action.yml | 38 +++++++++++++++++++++ .github/workflows/build.yml | 39 ++++------------------ .github/workflows/functional.yaml | 6 +--- .github/workflows/push-container.yml | 2 -- 5 files changed, 84 insertions(+), 39 deletions(-) create mode 100644 .github/actions/compile-aarch64/action.yml create mode 100644 .github/actions/compile-x86_64/action.yml diff --git a/.github/actions/compile-aarch64/action.yml b/.github/actions/compile-aarch64/action.yml new file mode 100644 index 000000000..58eefdb7a --- /dev/null +++ b/.github/actions/compile-aarch64/action.yml @@ -0,0 +1,38 @@ +name: "Compile Rust aarch64" +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + rust/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - uses: ./.github/actions/setup-rust + - run: cargo install cross || true + - run: CROSS_CONFIG=Cross.toml cross -v build --release --target aarch64-unknown-linux-gnu + working-directory: rust + - name: "patch for debian stable" + working-directory: rust + run: | + find . -type f -name "scannerctl" + patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/aarch64-unknown-linux-gnu/release/scannerctl + patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/x86_64-unknown-linux-gnu/release/scannerctl + patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl + patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl + - run: mkdir assets/ + - run: mkdir -p assets/linux/arm64 + - run: mv rust/target/aarch64-unknown-linux-gnu/release/openvasd assets/linux/arm64/openvasd + - run: mv rust/target/aarch64-unknown-linux-gnu/release/scannerctl assets/linux/arm64/scannerctl + - run: mv rust/target/aarch64-unknown-linux-gnu/release/feed-verifier assets/linux/arm64/feed-verifier + - name: archive + uses: actions/upload-artifact@v4 + with: + name: rs-binaries + path: assets/* + retention-days: 1 diff --git a/.github/actions/compile-x86_64/action.yml b/.github/actions/compile-x86_64/action.yml new file mode 100644 index 000000000..a5a24d413 --- /dev/null +++ b/.github/actions/compile-x86_64/action.yml @@ -0,0 +1,38 @@ +name: "Compile Rust x86_64" +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + rust/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + # TODO: probably the upload artifact is not actually needed + - uses: ./.github/actions/setup-rust + - run: cargo install cross || true + - run: CROSS_CONFIG=Cross.toml cross build --release --target x86_64-unknown-linux-gnu + working-directory: rust + - name: "patch for debian stable" + working-directory: rust + run: | + find . -type f -name "scannerctl" + patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/aarch64-unknown-linux-gnu/release/scannerctl + patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/x86_64-unknown-linux-gnu/release/scannerctl + patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl + patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl + - run: mkdir assets/ + - run: mkdir -p assets/linux/amd64 + - run: mv rust/target/x86_64-unknown-linux-gnu/release/openvasd assets/linux/amd64/openvasd + - run: mv rust/target/x86_64-unknown-linux-gnu/release/scannerctl assets/linux/amd64/scannerctl + - run: mv rust/target/x86_64-unknown-linux-gnu/release/feed-verifier assets/linux/amd64/feed-verifier + - name: archive + uses: actions/upload-artifact@v4 + with: + name: rs-binaries + path: assets/* + retention-days: 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3927c278f..f082744bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,9 +15,6 @@ jobs: run: | cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer cmake --build build - # TODO: change that to cargo check to check if it would be buildable but - # move the actual when it is needed - mainly on release - # This could significantly improve the speed of the CI Rust: runs-on: "ubuntu-latest" steps: @@ -31,33 +28,11 @@ jobs: ~/.cargo/git/db/ rust/target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: rustup update stable && rustup default stable - # ignore failing install, it may already be installed - - run: cargo install cross || true - - run: CROSS_CONFIG=Cross.toml cross -v build --release --target aarch64-unknown-linux-gnu - working-directory: rust - - run: CROSS_CONFIG=Cross.toml cross build --release --target x86_64-unknown-linux-gnu - working-directory: rust - - name: "patch for debian stable" + + - uses: ./.github/actions/setup-rust + # we just check if it is compilable, the actual compilation is now done in: + # - in the Dockefiles + # - in functional.yml (x86_64) only + # - in release.yml (aarch64, x86_64) + - run: cargo check working-directory: rust - run: | - find . -type f -name "scannerctl" - patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/aarch64-unknown-linux-gnu/release/scannerctl - patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/x86_64-unknown-linux-gnu/release/scannerctl - patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl - patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl - - run: mkdir assets/ - - run: mkdir -p assets/linux/amd64 - - run: mkdir -p assets/linux/arm64 - - run: mv rust/target/aarch64-unknown-linux-gnu/release/openvasd assets/linux/arm64/openvasd - - run: mv rust/target/x86_64-unknown-linux-gnu/release/openvasd assets/linux/amd64/openvasd - - run: mv rust/target/aarch64-unknown-linux-gnu/release/scannerctl assets/linux/arm64/scannerctl - - run: mv rust/target/x86_64-unknown-linux-gnu/release/scannerctl assets/linux/amd64/scannerctl - - run: mv rust/target/aarch64-unknown-linux-gnu/release/feed-verifier assets/linux/arm64/feed-verifier - - run: mv rust/target/x86_64-unknown-linux-gnu/release/feed-verifier assets/linux/amd64/feed-verifier - - name: archive - uses: actions/upload-artifact@v4 - with: - name: rs-binaries - path: assets/* - retention-days: 1 diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 3ee422a0d..683c397ff 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -45,11 +45,7 @@ jobs: run: | cmake -Bbuild -DCMAKE_BUILD_TYPE=Release cmake --build build -- install - # TODO: build instead a x86_64 version - - uses: actions/download-artifact@v4 - with: - name: rs-binaries - path: assets + - uses: ./.github/actions/compile-x86_64 - name: prepare setup run: | apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ diff --git a/.github/workflows/push-container.yml b/.github/workflows/push-container.yml index 45dfd08d9..45db1d7fc 100644 --- a/.github/workflows/push-container.yml +++ b/.github/workflows/push-container.yml @@ -79,8 +79,6 @@ jobs: registry: ${{ vars.IMAGE_REGISTRY }} registry-username: ${{ github.actor }} registry-password: ${{ secrets.GITHUB_TOKEN }} - scout-user: ${{ secrets.dockerhub_user }} - scout-password: ${{ secrets.dockerhub_token }} debian_stable: name: ghcr:debian:stable From 3360084557c6899e645b7ff9d7754a1aab3f34dd Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 11:08:28 +0100 Subject: [PATCH 07/11] CI: remove unittests dependency for build Removes unittests dependency on build to let the build and unit test run concurrently. Add unittests dependency for functional tests as it doesn't make sense to run functional tests when the unit tests fail. Removes functional dependency for container as it is not a requirement for non release container to be functional. --- .github/workflows/control.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/control.yml b/.github/workflows/control.yml index 66c02d748..300ca871e 100644 --- a/.github/workflows/control.yml +++ b/.github/workflows/control.yml @@ -75,15 +75,14 @@ jobs: name: unit-tests uses: ./.github/workflows/tests.yml build: - needs: [unittests] uses: ./.github/workflows/build.yml linting: uses: ./.github/workflows/linting.yml functional: - needs: [build] + needs: [unittests, build] uses: ./.github/workflows/functional.yaml container: - needs: [build, init, functional] + needs: [build, init] uses: ./.github/workflows/push-container.yml secrets: dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} From c8e85bae897d3c52d21e79ccc3ae38ab84fe7460 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 11:12:30 +0100 Subject: [PATCH 08/11] Adds bash shell into compile actions --- .github/actions/compile-aarch64/action.yml | 7 +++++++ .github/actions/compile-x86_64/action.yml | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/.github/actions/compile-aarch64/action.yml b/.github/actions/compile-aarch64/action.yml index 58eefdb7a..680547147 100644 --- a/.github/actions/compile-aarch64/action.yml +++ b/.github/actions/compile-aarch64/action.yml @@ -15,8 +15,10 @@ runs: - uses: ./.github/actions/setup-rust - run: cargo install cross || true + shell: bash - run: CROSS_CONFIG=Cross.toml cross -v build --release --target aarch64-unknown-linux-gnu working-directory: rust + shell: bash - name: "patch for debian stable" working-directory: rust run: | @@ -26,10 +28,15 @@ runs: patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl - run: mkdir assets/ + shell: bash - run: mkdir -p assets/linux/arm64 + shell: bash - run: mv rust/target/aarch64-unknown-linux-gnu/release/openvasd assets/linux/arm64/openvasd + shell: bash - run: mv rust/target/aarch64-unknown-linux-gnu/release/scannerctl assets/linux/arm64/scannerctl + shell: bash - run: mv rust/target/aarch64-unknown-linux-gnu/release/feed-verifier assets/linux/arm64/feed-verifier + shell: bash - name: archive uses: actions/upload-artifact@v4 with: diff --git a/.github/actions/compile-x86_64/action.yml b/.github/actions/compile-x86_64/action.yml index a5a24d413..8cee3f183 100644 --- a/.github/actions/compile-x86_64/action.yml +++ b/.github/actions/compile-x86_64/action.yml @@ -15,10 +15,13 @@ runs: # TODO: probably the upload artifact is not actually needed - uses: ./.github/actions/setup-rust - run: cargo install cross || true + shell: bash - run: CROSS_CONFIG=Cross.toml cross build --release --target x86_64-unknown-linux-gnu working-directory: rust + shell: bash - name: "patch for debian stable" working-directory: rust + shell: bash run: | find . -type f -name "scannerctl" patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/aarch64-unknown-linux-gnu/release/scannerctl @@ -26,10 +29,15 @@ runs: patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl - run: mkdir assets/ + shell: bash - run: mkdir -p assets/linux/amd64 + shell: bash - run: mv rust/target/x86_64-unknown-linux-gnu/release/openvasd assets/linux/amd64/openvasd + shell: bash - run: mv rust/target/x86_64-unknown-linux-gnu/release/scannerctl assets/linux/amd64/scannerctl + shell: bash - run: mv rust/target/x86_64-unknown-linux-gnu/release/feed-verifier assets/linux/amd64/feed-verifier + shell: bash - name: archive uses: actions/upload-artifact@v4 with: From fd3d893817e3447bd1a2dd20a293be955d8b4ed6 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 11:25:14 +0100 Subject: [PATCH 09/11] Move rust cache into setup-rust --- .github/actions/compile-aarch64/action.yml | 11 ----------- .github/actions/compile-x86_64/action.yml | 11 ----------- .github/actions/setup-rust/action.yml | 9 +++++++++ .github/workflows/build.yml | 10 ---------- 4 files changed, 9 insertions(+), 32 deletions(-) diff --git a/.github/actions/compile-aarch64/action.yml b/.github/actions/compile-aarch64/action.yml index 680547147..2eaefa989 100644 --- a/.github/actions/compile-aarch64/action.yml +++ b/.github/actions/compile-aarch64/action.yml @@ -2,17 +2,6 @@ name: "Compile Rust aarch64" runs: using: "composite" steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - rust/target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - uses: ./.github/actions/setup-rust - run: cargo install cross || true shell: bash diff --git a/.github/actions/compile-x86_64/action.yml b/.github/actions/compile-x86_64/action.yml index 8cee3f183..9da415bdc 100644 --- a/.github/actions/compile-x86_64/action.yml +++ b/.github/actions/compile-x86_64/action.yml @@ -2,17 +2,6 @@ name: "Compile Rust x86_64" runs: using: "composite" steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - rust/target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # TODO: probably the upload artifact is not actually needed - uses: ./.github/actions/setup-rust - run: cargo install cross || true shell: bash diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index be188d042..21e5cf184 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -3,6 +3,15 @@ description: "Install necessary dependencies and set up Rust stable" runs: using: "composite" steps: + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + rust/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('rust/Cargo.lock') }} - run: sudo apt update || true shell: bash - run: sudo apt-get install -y libpcap-dev diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f082744bd..65ff32e1a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,16 +19,6 @@ jobs: runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - rust/target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - uses: ./.github/actions/setup-rust # we just check if it is compilable, the actual compilation is now done in: # - in the Dockefiles From fce5abc06a6b98aebed00e264ca2c8df4664ba5d Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 12:35:55 +0100 Subject: [PATCH 10/11] CI: functional tests: download previously build binaries As functional tests are running within a gvm-libs image it needs to exec compile workflows before starting the test cases. This ensures that the binary build by compile workflow is actually working on our image. --- .github/workflows/functional.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 683c397ff..c87e01717 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -22,9 +22,14 @@ jobs: - run: docker run --rm test ldd /usr/local/sbin/openvas - run: docker run --rm test ldd /usr/local/sbin/openvas | grep libopenvas_wmiclient - run: docker rmi test || true - # TESTS that are possible before pushing an image + build-rs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/compile-x86_64 tests: runs-on: ubuntu-latest + needs: [build-rs] services: redis: image: redis @@ -45,7 +50,10 @@ jobs: run: | cmake -Bbuild -DCMAKE_BUILD_TYPE=Release cmake --build build -- install - - uses: ./.github/actions/compile-x86_64 + - uses: actions/download-artifact@v4 + with: + name: rs-binaries + path: assets - name: prepare setup run: | apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ From ab26865c36331ea8bd1279761a119f5f79bfc535 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 6 Feb 2025 15:24:55 +0100 Subject: [PATCH 11/11] Fixes patchelf path --- .github/actions/compile-aarch64/action.yml | 2 -- .github/actions/compile-x86_64/action.yml | 2 -- .github/workflows/linting.yml | 4 ++-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/compile-aarch64/action.yml b/.github/actions/compile-aarch64/action.yml index 2eaefa989..46400c365 100644 --- a/.github/actions/compile-aarch64/action.yml +++ b/.github/actions/compile-aarch64/action.yml @@ -13,9 +13,7 @@ runs: run: | find . -type f -name "scannerctl" patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/aarch64-unknown-linux-gnu/release/scannerctl - patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/x86_64-unknown-linux-gnu/release/scannerctl patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl - patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl - run: mkdir assets/ shell: bash - run: mkdir -p assets/linux/arm64 diff --git a/.github/actions/compile-x86_64/action.yml b/.github/actions/compile-x86_64/action.yml index 9da415bdc..edb2da700 100644 --- a/.github/actions/compile-x86_64/action.yml +++ b/.github/actions/compile-x86_64/action.yml @@ -13,9 +13,7 @@ runs: shell: bash run: | find . -type f -name "scannerctl" - patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/aarch64-unknown-linux-gnu/release/scannerctl patchelf --replace-needed libpcap.so.1 libpcap.so.0.8 target/x86_64-unknown-linux-gnu/release/scannerctl - patchelf --replace-needed libz.so libz.so.1 target/aarch64-unknown-linux-gnu/release/scannerctl patchelf --replace-needed libz.so libz.so.1 target/x86_64-unknown-linux-gnu/release/scannerctl - run: mkdir assets/ shell: bash diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 78d2cdd40..add88ebb5 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-rust - - run: cargo install typos-cli + - run: cargo install typos-cli || true - run: typos Rust-Audit: runs-on: ubuntu-latest @@ -37,7 +37,7 @@ jobs: working-directory: rust steps: - uses: actions/checkout@v4 - - run: cargo install cargo-audit + - run: cargo install cargo-audit || true - run: cargo audit License-Headers: runs-on: ubuntu-latest