From 00b9424c149c6da69b11f486ed9ac60076a443ec Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 11 Feb 2025 15:42:44 +0100 Subject: [PATCH 01/10] Refactor: use greenbone registry, rename ws dirs --- .github/workflows/smoketest.yaml | 6 +++--- charts/.gitignore | 2 ++ charts/Makefile | 1 + charts/openvasd/values.yaml | 14 +++++++------- .../{PKI mTLS Method => pki}/ca_certificates.sh | 0 .../client_certificates.sh | 3 ++- .../server_certificates.sh | 4 +++- .../Makefile | 0 .../client_certificates.sh | 5 +++-- .../server_certificates.sh | 5 +++-- rust/src/openvasd/README.md | 8 ++++---- 11 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 charts/.gitignore rename rust/examples/tls/{PKI mTLS Method => pki}/ca_certificates.sh (100%) rename rust/examples/tls/{PKI mTLS Method => pki}/client_certificates.sh (85%) rename rust/examples/tls/{PKI mTLS Method => pki}/server_certificates.sh (84%) rename rust/examples/tls/{Self-Signed mTLS Method => self-signed}/Makefile (100%) rename rust/examples/tls/{Self-Signed mTLS Method => self-signed}/client_certificates.sh (87%) rename rust/examples/tls/{Self-Signed mTLS Method => self-signed}/server_certificates.sh (87%) diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml index a36ddfae6..35aa35f9e 100644 --- a/.github/workflows/smoketest.yaml +++ b/.github/workflows/smoketest.yaml @@ -32,7 +32,7 @@ jobs: metrics-enabled: false - name: deploy openvasd run: | - cd rust/examples/tls/Self-Signed\ mTLS\ Method + cd rust/examples/tls/self-signed make delete deploy cd - @@ -45,7 +45,7 @@ jobs: working-directory: rust/crates/smoketest env: SCAN_CONFIG: configs/simple_scan_ssh_only.json - CLIENT_KEY: ../examples/tls/Self-Signed mTLS Method/client.rsa - CLIENT_CERT: ../examples/tls/Self-Signed mTLS Method/client.pem + CLIENT_KEY: ../examples/tls/self-signed/client.rsa + CLIENT_CERT: ../examples/tls/self-signed/client.pem run: | make build run diff --git a/charts/.gitignore b/charts/.gitignore new file mode 100644 index 000000000..a26494ded --- /dev/null +++ b/charts/.gitignore @@ -0,0 +1,2 @@ +*.rsa +*.pem diff --git a/charts/Makefile b/charts/Makefile index 7e4e676e3..08d6da5cc 100644 --- a/charts/Makefile +++ b/charts/Makefile @@ -1,4 +1,5 @@ _PHONY: install-http install-mtls uninstall log-openvasd +TLS_PATH="../rust/examples/tls/self-signed" install-http: helm install --namespace openvasd --create-namespace openvasd openvasd/ --values openvasd/values.yaml --values openvasd/http-root.yaml diff --git a/charts/openvasd/values.yaml b/charts/openvasd/values.yaml index abd72460e..4cabd0e6b 100644 --- a/charts/openvasd/values.yaml +++ b/charts/openvasd/values.yaml @@ -4,13 +4,13 @@ # Is currently reqired by openvasd to control openvas ospd: - repository: greenbone/ospd-openvas + repository: registry.community.greenbone.net/community/ospd-openvas pullPolicy: Always tag: "edge" # Contains openvasd openvas: - repository: greenbone/openvas-scanner + repository: registry.community.greenbone.net/community/openvas-scanner pullPolicy: Always tag: "edge" @@ -44,33 +44,33 @@ openvasd: # Required for version checks notus: - repository: greenbone/notus-scanner + repository: registry.community.greenbone.net/community/notus-scanner pullPolicy: Always tag: "edge" # NASL scripts also known as feed vulnerabilitytests: # latest is the most current community feed. - repository: greenbone/vulnerability-tests + repository: registry.community.greenbone.net/community/vulnerability-tests pullPolicy: Always tag: "latest" # Notus description json also known as feed notusdata: # latest is the most current community feed. - repository: greenbone/notus-data + repository: registry.community.greenbone.net/community/notus-data pullPolicy: Always tag: "latest" # Required by notus mqtt: - repository: greenbone/mqtt-broker + repository: registry.community.greenbone.net/community/mqtt-broker pullPolicy: Always tag: "latest" # required by openvas and ospd redis: - repository: greenbone/redis-server + repository: registry.community.greenbone.net/community/redis-server pullPolicy: Always tag: "latest" diff --git a/rust/examples/tls/PKI mTLS Method/ca_certificates.sh b/rust/examples/tls/pki/ca_certificates.sh similarity index 100% rename from rust/examples/tls/PKI mTLS Method/ca_certificates.sh rename to rust/examples/tls/pki/ca_certificates.sh diff --git a/rust/examples/tls/PKI mTLS Method/client_certificates.sh b/rust/examples/tls/pki/client_certificates.sh similarity index 85% rename from rust/examples/tls/PKI mTLS Method/client_certificates.sh rename to rust/examples/tls/pki/client_certificates.sh index 52bd2cfdf..42518e912 100644 --- a/rust/examples/tls/PKI mTLS Method/client_certificates.sh +++ b/rust/examples/tls/pki/client_certificates.sh @@ -4,6 +4,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later WITH x11vnc-openssl-exception set -xe +dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) openssl req -nodes \ -newkey rsa:2048 \ @@ -25,6 +26,6 @@ openssl x509 -req \ -sha256 \ -days 2000 \ -set_serial 456 \ - -extensions v3_end -extfile ../openssl.cnf + -extensions v3_end -extfile $dir/openssl.cnf mv end.cert client.cert diff --git a/rust/examples/tls/PKI mTLS Method/server_certificates.sh b/rust/examples/tls/pki/server_certificates.sh similarity index 84% rename from rust/examples/tls/PKI mTLS Method/server_certificates.sh rename to rust/examples/tls/pki/server_certificates.sh index 397eb4c85..4576e8deb 100644 --- a/rust/examples/tls/PKI mTLS Method/server_certificates.sh +++ b/rust/examples/tls/pki/server_certificates.sh @@ -3,6 +3,8 @@ # # SPDX-License-Identifier: GPL-2.0-or-later WITH x11vnc-openssl-exception +dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) + set -xe openssl req -nodes \ @@ -25,7 +27,7 @@ openssl x509 -req \ -sha256 \ -days 2000 \ -set_serial 456 \ - -extensions v3_end -extfile ../openssl.cnf + -extensions v3_end -extfile "$dir/openssl.cnf" cp end.cert server.cert diff --git a/rust/examples/tls/Self-Signed mTLS Method/Makefile b/rust/examples/tls/self-signed/Makefile similarity index 100% rename from rust/examples/tls/Self-Signed mTLS Method/Makefile rename to rust/examples/tls/self-signed/Makefile diff --git a/rust/examples/tls/Self-Signed mTLS Method/client_certificates.sh b/rust/examples/tls/self-signed/client_certificates.sh similarity index 87% rename from rust/examples/tls/Self-Signed mTLS Method/client_certificates.sh rename to rust/examples/tls/self-signed/client_certificates.sh index b0bcb286d..a796245ab 100644 --- a/rust/examples/tls/Self-Signed mTLS Method/client_certificates.sh +++ b/rust/examples/tls/self-signed/client_certificates.sh @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later WITH x11vnc-openssl-exception +dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) set -xe openssl req -nodes \ @@ -43,7 +44,7 @@ openssl x509 -req \ -sha256 \ -days 3650 \ -set_serial 123 \ - -extensions v3_inter -extfile ../openssl.cnf + -extensions v3_inter -extfile "$dir/../openssl.cnf" openssl x509 -req \ -in end.req \ @@ -53,7 +54,7 @@ openssl x509 -req \ -sha256 \ -days 2000 \ -set_serial 456 \ - -extensions v3_end -extfile ../openssl.cnf + -extensions v3_end -extfile "$dir/../openssl.cnf" cat end.cert inter.cert ca.cert > client.pem rm *.key *.cert *.req diff --git a/rust/examples/tls/Self-Signed mTLS Method/server_certificates.sh b/rust/examples/tls/self-signed/server_certificates.sh similarity index 87% rename from rust/examples/tls/Self-Signed mTLS Method/server_certificates.sh rename to rust/examples/tls/self-signed/server_certificates.sh index c52af7795..698f88001 100644 --- a/rust/examples/tls/Self-Signed mTLS Method/server_certificates.sh +++ b/rust/examples/tls/self-signed/server_certificates.sh @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later WITH x11vnc-openssl-exception +dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) set -xe openssl req -nodes \ @@ -43,7 +44,7 @@ openssl x509 -req \ -sha256 \ -days 3650 \ -set_serial 123 \ - -extensions v3_inter -extfile ../openssl.cnf + -extensions v3_inter -extfile "$dir/../openssl.cnf" openssl x509 -req \ -in end.req \ @@ -53,7 +54,7 @@ openssl x509 -req \ -sha256 \ -days 2000 \ -set_serial 456 \ - -extensions v3_end -extfile ../openssl.cnf + -extensions v3_end -extfile "$dir/../openssl.cnf" cat end.cert inter.cert ca.cert > server.pem rm *.key *.cert *.req diff --git a/rust/src/openvasd/README.md b/rust/src/openvasd/README.md index 569210d48..ce163ad10 100644 --- a/rust/src/openvasd/README.md +++ b/rust/src/openvasd/README.md @@ -11,7 +11,7 @@ Currently it is utilizing ospd-openvas. - [API Key](#api-key) - [Certificates](#certificates) - [How does mTLS works?](#how-does-mtls-works) - - [Certificate Authority for PKI mTLS Method](#certificate-authority-for-pki-mtls-method) + - [Certificate Authority for pki](#certificate-authority-for-pki-mtls-method) - [mTLS with self-signed client certificates.](#mtls-with-self-signed-client-certificates) - [Mode](#mode) - [Usage](#usage) @@ -85,13 +85,13 @@ It is similar to the known TLS, but there are some extra steps. In the following 6. **The server gives access to the client** 7. Client and server exchange information through a encrypted TLS connexion. -#### Certificate Authority for PKI mTLS Method +#### Certificate Authority for pki Who implements mTLS is its own Certification Authority. This is a difference from standard TLS, where an external organization verifies and validates the TLS certificates. For mTLS is necessary a TLS root certificate which allows an organization to be its owns certificate authority. The root CA signs intermediate certificates forming a chain of trust. The intermediate CA is created for security reasons. then, the root certificate is used as less as possible and in case the intermediate key is compromised, the root CA can revoke the intermediate key and create a new one. Finally, the intermediate CA is used to sign client and server certificates. -In the examples folder for a [PKI Mutual TLS](../examples/tls/PKI mTLS Method) you can find three scripts for creation of root and intermediate CA, client and server pairs (keys and certs). +In the examples folder for a [PKI Mutual TLS](../examples/tls/pki) you can find three scripts for creation of root and intermediate CA, client and server pairs (keys and certs). Once you created the all key-certs pairs, you can use them for stablishing a mTLS secure connexion between `openvasd` and the clients. @@ -114,7 +114,7 @@ This method is similar to the explained above, but differs in that the clients a Then, the clients have to register a single certificate with the authorization server, and there is no shared CA certificate. During authentication, the server checks if the client uses the same certificate for the TLS session as was configured or registered for that individual client. The server trusts the pinned certificate. The client's organization does not have to maintain any public key infrastructure and can simply use a self-signed certificate for authentication. -The scripts for generating the self-signed certificate can be found in [Self-Signed mTLS Method](../examples/tls/Self-Signed mTLS Method) folder. +The scripts for generating the self-signed certificate can be found in [self-signed](../examples/tls/self-signed) folder. Once you created the all key-certs pairs, you can use them for stablishing a mTLS secure connexion between `openvasd` and the clients. From 26d9e429d2bb15f304308175436008d6d705c185 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 12 Feb 2025 02:48:26 +0100 Subject: [PATCH 02/10] Remove build.Dockerfile --- .docker/build.Dockerfile | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 .docker/build.Dockerfile diff --git a/.docker/build.Dockerfile b/.docker/build.Dockerfile deleted file mode 100644 index 2e11844a9..000000000 --- a/.docker/build.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -ARG VERSION=edge - -FROM registry.community.greenbone.net/community/gvm-libs:${VERSION} -LABEL deprecated="This image is deprecated and may be removed soon." - -RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ - bison \ - build-essential \ - clang \ - clang-format \ - clang-tools \ - cmake \ - lcov \ - libgnutls28-dev \ - libgpgme-dev \ - libjson-glib-dev \ - libksba-dev \ - libpaho-mqtt-dev \ - libpcap-dev \ - zlib1g-dev \ - libssh-gcrypt-dev \ - libbsd-dev \ - # for linking openvas-smb (libopenvas_wmiclient) - libgnutls30 \ - libgssapi3-heimdal \ - libkrb5-26-heimdal \ - libasn1-8-heimdal \ - libroken19-heimdal \ - libhdb9-heimdal \ - libpopt0 \ - libcurl4-gnutls-dev \ - && rm -rf /var/lib/apt/lists/* From 257fbc5bff38c066c606ac7be9721c2841e24bab Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 12 Feb 2025 03:05:29 +0100 Subject: [PATCH 03/10] Move smoketest into functional.yaml This enabled us to verify the helm chart and smoketests on each PR rather than having to wait until it is merged. --- .github/workflows/control.yml | 2 +- .github/workflows/functional.yaml | 44 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/control.yml b/.github/workflows/control.yml index 300ca871e..6cc6381d5 100644 --- a/.github/workflows/control.yml +++ b/.github/workflows/control.yml @@ -79,7 +79,7 @@ jobs: linting: uses: ./.github/workflows/linting.yml functional: - needs: [unittests, build] + # needs: [unittests, build] uses: ./.github/workflows/functional.yaml container: needs: [build, init] diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index c87e01717..f0daf1290 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -27,6 +27,50 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/compile-x86_64 + + smoketest: + runs-on: ubuntu-latest + steps: + - name: Start a local k8s cluster + uses: jupyterhub/action-k3s-helm@v4 + with: + k3s-channel: latest + metrics-enabled: false + docker-enabled: true + - uses: actions/checkout@v4 + - name: publish current docker image into k3s + run: | + docker build -t greenbone/openvas:pr -f .docker/prod.Dockerfile . + - name: 'Set openvas-scanner-image' + run: | + echo "openvas:" >> st.yaml + echo " repository: greenbone/openvas" >> st.yaml + echo " pullPolicy: Never" >> st.yaml + echo " tag: \"pr\"" >> st.yaml + cat st.yaml + - name: deploy openvasd + run: | + cd rust/examples/tls/self-signed + make delete deploy + cd - + + helm uninstall openvasd --namespace openvasd|| true + helm install --namespace openvasd --create-namespace openvasd charts/openvasd/ --values charts/openvasd/values.yaml --values charts/openvasd/mtls-wo-ingress.yaml --values st.yaml + + kubectl rollout status --watch --timeout 600s deployment/openvasd --namespace openvasd + echo "OPENVASD_SERVER=https://$(kubectl get svc -n openvasd | awk 'FNR == 2 {print $(3)}')" >> $GITHUB_ENV + - name: smoketest + working-directory: rust/crates/smoketest + env: + SCAN_CONFIG: configs/simple_scan_ssh_only.json + CLIENT_KEY: ../../../examples/tls/self-signed/client.rsa + CLIENT_CERT: ../../../examples/tls/self-signed/client.pem + run: | + make build run + # - uses: actions/cache@v4 + # with: + # path: /tmp/openvas.tar + # key: openvas-cache-${{ github.run_id }} tests: runs-on: ubuntu-latest needs: [build-rs] From 6846363ce3ae400c713af867e1413f4e36124a86 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 13 Feb 2025 10:08:31 +0100 Subject: [PATCH 04/10] Switches from ubuntu-latest to self-hosted-generic --- .github/workflows/auto_label.yml | 2 +- .github/workflows/build.yml | 4 ++-- .github/workflows/codeql.yml | 2 +- .github/workflows/control.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/functional.yaml | 8 ++++---- .github/workflows/helm-release-on-tag.yml | 2 +- .github/workflows/init.yaml | 2 +- .github/workflows/linting.yml | 10 +++++----- .github/workflows/push-container-oldstable.yml | 2 +- .github/workflows/push-container-testing.yml | 2 +- .github/workflows/push-container.yml | 2 +- .github/workflows/push-helm-chart.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/sbom-upload.yml | 2 +- .github/workflows/smoketest.yaml | 2 +- .github/workflows/tests.yml | 4 ++-- 17 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/auto_label.yml b/.github/workflows/auto_label.yml index ab9771ea6..962118d54 100644 --- a/.github/workflows/auto_label.yml +++ b/.github/workflows/auto_label.yml @@ -9,7 +9,7 @@ permissions: jobs: label: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - uses: greenbone/actions/pr-conventional-commit-labeler@main with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65ff32e1a..02a5e4cd3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: [workflow_call] jobs: C: - runs-on: ubuntu-latest + runs-on: self-hosted-generic container: greenbone/gvm-libs:stable steps: - uses: actions/checkout@v4 @@ -16,7 +16,7 @@ jobs: cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer cmake --build build Rust: - runs-on: "ubuntu-latest" + runs-on: "self-hosted-generic" steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-rust diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 13f7a303e..85b870b63 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -11,7 +11,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: self-hosted-generic permissions: actions: read contents: read diff --git a/.github/workflows/control.yml b/.github/workflows/control.yml index 6cc6381d5..4f5861a83 100644 --- a/.github/workflows/control.yml +++ b/.github/workflows/control.yml @@ -40,7 +40,7 @@ jobs: # this prevents us from having to pass down all labels, event_name, etc # to init.yml adapt_release: - runs-on: ubuntu-latest + runs-on: self-hosted-generic outputs: kind: ${{ steps.kind.outputs.kind}} steps: diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index d4be39743..17d60f8f8 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -7,7 +7,7 @@ permissions: jobs: dependency-review: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - name: 'Dependency Review' uses: greenbone/actions/dependency-review@v3 diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index f0daf1290..5b0148783 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -10,7 +10,7 @@ jobs: # Tests that gvm-libs, openvas-smb and openvas dependencies work together and # that openvas is buildable and integrates openvas-smb when available distributed-monolith-railguard: - runs-on: ubuntu-latest + runs-on: self-hosted-generic strategy: fail-fast: false matrix: @@ -23,13 +23,13 @@ jobs: - run: docker run --rm test ldd /usr/local/sbin/openvas | grep libopenvas_wmiclient - run: docker rmi test || true build-rs: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - uses: actions/checkout@v4 - uses: ./.github/actions/compile-x86_64 smoketest: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - name: Start a local k8s cluster uses: jupyterhub/action-k3s-helm@v4 @@ -72,7 +72,7 @@ jobs: # path: /tmp/openvas.tar # key: openvas-cache-${{ github.run_id }} tests: - runs-on: ubuntu-latest + runs-on: self-hosted-generic needs: [build-rs] services: redis: diff --git a/.github/workflows/helm-release-on-tag.yml b/.github/workflows/helm-release-on-tag.yml index 01d54d4bd..6c4173788 100644 --- a/.github/workflows/helm-release-on-tag.yml +++ b/.github/workflows/helm-release-on-tag.yml @@ -7,7 +7,7 @@ on: jobs: release-helm-chart: name: Release helm chart - runs-on: ubuntu-latest + runs-on: self-hosted-generic strategy: fail-fast: false matrix: diff --git a/.github/workflows/init.yaml b/.github/workflows/init.yaml index a9dd5dd26..9663f192f 100644 --- a/.github/workflows/init.yaml +++ b/.github/workflows/init.yaml @@ -45,7 +45,7 @@ on: jobs: init: - runs-on: ubuntu-latest + runs-on: self-hosted-generic outputs: is_latest_tag: ${{ steps.version.outputs.is_latest_tag }} is_version_tag: ${{ steps.version.outputs.is_version_tag }} diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index add88ebb5..cf155b7df 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -4,14 +4,14 @@ on: [workflow_call] jobs: C: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - name: Check out openvas-scanner uses: actions/checkout@v4 - name: Formatting run: sh .github/check-c-formatting.sh ${{ github.base_ref }} Rust: - runs-on: ubuntu-latest + runs-on: self-hosted-generic defaults: run: working-directory: rust @@ -21,7 +21,7 @@ jobs: - run: cargo clippy -- -D warnings - run: cargo fmt --check Rust-Typos: - runs-on: ubuntu-latest + runs-on: self-hosted-generic defaults: run: working-directory: rust @@ -31,7 +31,7 @@ jobs: - run: cargo install typos-cli || true - run: typos Rust-Audit: - runs-on: ubuntu-latest + runs-on: self-hosted-generic defaults: run: working-directory: rust @@ -40,7 +40,7 @@ jobs: - run: cargo install cargo-audit || true - run: cargo audit License-Headers: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - name: Check out openvas-scanner uses: actions/checkout@v4 diff --git a/.github/workflows/push-container-oldstable.yml b/.github/workflows/push-container-oldstable.yml index c859f38d0..7cd2ae3ba 100644 --- a/.github/workflows/push-container-oldstable.yml +++ b/.github/workflows/push-container-oldstable.yml @@ -18,7 +18,7 @@ on: jobs: debian_oldstable: name: ghcr:debian:oldstable - runs-on: "ubuntu-latest" + runs-on: "self-hosted-generic" steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/push-container-testing.yml b/.github/workflows/push-container-testing.yml index d17a463e7..dd90c5e84 100644 --- a/.github/workflows/push-container-testing.yml +++ b/.github/workflows/push-container-testing.yml @@ -19,7 +19,7 @@ jobs: # TODO: do we need to push or is building enough? debian_testing: name: ghcr:debian:testing - runs-on: "ubuntu-latest" + runs-on: "self-hosted-generic" steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/push-container.yml b/.github/workflows/push-container.yml index 45db1d7fc..30e893936 100644 --- a/.github/workflows/push-container.yml +++ b/.github/workflows/push-container.yml @@ -82,7 +82,7 @@ jobs: debian_stable: name: ghcr:debian:stable - runs-on: "ubuntu-latest" + runs-on: "self-hosted-generic" steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/push-helm-chart.yml b/.github/workflows/push-helm-chart.yml index 5d638e56b..4cdcf6e2b 100644 --- a/.github/workflows/push-helm-chart.yml +++ b/.github/workflows/push-helm-chart.yml @@ -15,7 +15,7 @@ on: jobs: helm: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - uses: actions/checkout@v4 - uses: greenbone/actions/helm-build-push@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c3ea8981f..807524797 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ on: jobs: release: name: release - runs-on: "ubuntu-latest" + runs-on: "self-hosted-generic" env: RELEASE_KIND: ${{inputs.release_kind}} RELEASE_REF: ${{inputs.release_ref}} diff --git a/.github/workflows/sbom-upload.yml b/.github/workflows/sbom-upload.yml index fefdfeafe..f16b74704 100644 --- a/.github/workflows/sbom-upload.yml +++ b/.github/workflows/sbom-upload.yml @@ -5,7 +5,7 @@ on: branches: ["main"] jobs: SBOM-upload: - runs-on: ubuntu-latest + runs-on: self-hosted-generic permissions: id-token: write contents: write diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml index 35aa35f9e..66cff980f 100644 --- a/.github/workflows/smoketest.yaml +++ b/.github/workflows/smoketest.yaml @@ -15,7 +15,7 @@ on: # jobs: openvasd: - runs-on: ubuntu-latest + runs-on: self-hosted-generic steps: - uses: actions/checkout@v4 - name: 'Set openvas-scanner-image' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8c3b2aa85..0412103ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,7 @@ on: [workflow_call] jobs: C: - runs-on: ubuntu-latest + runs-on: self-hosted-generic container: greenbone/gvm-libs:stable steps: - uses: actions/checkout@v4 @@ -16,7 +16,7 @@ jobs: cmake -Bbuild -DCMAKE_BUILD_TYPE=Release CTEST_OUTPUT_ON_FAILURE=1 cmake --build build -- tests test Rust: - runs-on: ubuntu-latest + runs-on: self-hosted-generic defaults: run: working-directory: rust From bed12294d1e9a5ad7106d988a6a4160482c80f22 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 13 Feb 2025 14:58:04 +0100 Subject: [PATCH 05/10] Workaround for not able to start k3s Instead of running k3s on self-hosted-generic we run in ubuntu-latest and copy the image from an internal cache to a tar achieve to import those into the runner. --- .docker/prod.Dockerfile | 9 +++-- .github/workflows/functional.yaml | 47 ++++++++++++++++++++--- charts/openvasd/templates/deployment.yaml | 12 +----- charts/openvasd/values.yaml | 17 ++------ 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile index 81b0d373a..c900db43d 100644 --- a/.docker/prod.Dockerfile +++ b/.docker/prod.Dockerfile @@ -1,11 +1,12 @@ ARG VERSION=edge -# this allows to work on forked repository -ARG REPOSITORY=greenbone/openvas-scanner +# this allows to override gvm-libs for e.g. smoketests +ARG GVM_LIBS=registry.community.greenbone.net/community/gvm-libs FROM rust AS rust + FROM greenbone/openvas-smb AS openvas-smb -FROM registry.community.greenbone.net/community/gvm-libs:${VERSION} AS build +FROM ${GVM_LIBS}:${VERSION} AS build COPY . /source RUN sh /source/.github/install-openvas-dependencies.sh COPY --from=openvas-smb /usr/local/lib/ /usr/local/lib/ @@ -24,7 +25,7 @@ RUN cp target/release/scannerctl /install/usr/local/bin # Do we want to copy feed verifier as well? # RUN cp release/feed-verifier /install/bin -FROM registry.community.greenbone.net/community/gvm-libs:${VERSION} +FROM ${GVM_LIBS}:${VERSION} RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ bison \ libglib2.0-0 \ diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 5b0148783..938566c59 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -27,9 +27,31 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/compile-x86_64 + build-image: + runs-on: self-hosted-generic + steps: + - uses: actions/checkout@v4 + - run: | + docker build -t registry.community.greenbone.net/community/openvas-scanner:edge -f .docker/prod.Dockerfile . + docker pull registry.community.greenbone.net/community/vulnerability-tests:community + docker pull registry.community.greenbone.net/community/notus-data:community + docker save -o ${{ runner.temp }}/vtc.tar registry.community.greenbone.net/community/vulnerability-tests:community + docker save -o ${{ runner.temp }}/nc.tar registry.community.greenbone.net/community/notus-data:community + docker save -o ${{ runner.temp }}/openvas.tar registry.community.greenbone.net/community/openvas-scanner:edge + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ovimage + path: ${{ runner.temp }}/*.tar smoketest: - runs-on: self-hosted-generic + # currently we cannot use internal runner as they cannot start k3s until it + # is fixed we need to build the image in another job running on + # self-hosted-generic export it as a tar and import it here to work around + # the unreliability of registry.community.greenbone.net/community + # when called too often + runs-on: ubuntu-latest + needs: [build-image] steps: - name: Start a local k8s cluster uses: jupyterhub/action-k3s-helm@v4 @@ -38,15 +60,25 @@ jobs: metrics-enabled: false docker-enabled: true - uses: actions/checkout@v4 + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: ovimage + path: ${{ runner.temp }} - name: publish current docker image into k3s run: | - docker build -t greenbone/openvas:pr -f .docker/prod.Dockerfile . - - name: 'Set openvas-scanner-image' + docker load --input ${{ runner.temp }}/openvas.tar + docker load --input ${{ runner.temp }}/nc.tar + docker load --input ${{ runner.temp }}/vtc.tar + docker image ls -a + - name: 'do not, I repeat, do NOT pull that' run: | echo "openvas:" >> st.yaml - echo " repository: greenbone/openvas" >> st.yaml echo " pullPolicy: Never" >> st.yaml - echo " tag: \"pr\"" >> st.yaml + echo "vulnerabilitytests:" >> st.yaml + echo " pullPolicy: Never" >> st.yaml + echo "notusdata:" >> st.yaml + echo " pullPolicy: Never" >> st.yaml cat st.yaml - name: deploy openvasd run: | @@ -55,7 +87,10 @@ jobs: cd - helm uninstall openvasd --namespace openvasd|| true - helm install --namespace openvasd --create-namespace openvasd charts/openvasd/ --values charts/openvasd/values.yaml --values charts/openvasd/mtls-wo-ingress.yaml --values st.yaml + helm install --namespace openvasd --create-namespace openvasd charts/openvasd/ \ + --values charts/openvasd/values.yaml \ + --values charts/openvasd/mtls-wo-ingress.yaml \ + --values st.yaml kubectl rollout status --watch --timeout 600s deployment/openvasd --namespace openvasd echo "OPENVASD_SERVER=https://$(kubectl get svc -n openvasd | awk 'FNR == 2 {print $(3)}')" >> $GITHUB_ENV diff --git a/charts/openvasd/templates/deployment.yaml b/charts/openvasd/templates/deployment.yaml index 258f9ff9c..35afcbba4 100644 --- a/charts/openvasd/templates/deployment.yaml +++ b/charts/openvasd/templates/deployment.yaml @@ -83,6 +83,7 @@ spec: env: - name: MOUNT_PATH value: "/mnt/notus" + {{- if eq .Values.openvasd.scanner_type "ospd" }} - name: ospd-disable-notus-hashsum-verification #since can mount local volumes which may be altered we have to disable hashsum verification for notus image: "{{ .Values.ospd.repository }}:{{ .Values.ospd.tag }}" @@ -100,14 +101,6 @@ spec: mountPath: /mnt/ovc command: ['sh', '-c'] args: ["touch /mnt/ovc/openvas.log && chown ospd-openvas:ospd-openvas /mnt/ovc/openvas.log"] - - name: mqtt-broker-openvas-fix - image: "{{ .Values.ospd.repository }}:{{ .Values.ospd.tag }}" - imagePullPolicy: Always - volumeMounts: - - name: openvas-config - mountPath: /mnt/ovc - command: ['sh', '-c'] - args: ["sed 's/mqtt-broker/localhost/' /etc/openvas/openvas.conf > /mnt/ovc/openvas.conf; cp /etc/openvas/openvas_log.conf /mnt/ovc/"] {{ else }} - name: create-dummy-openvas-log image: "{{ .Values.openvas.repository }}:{{ .Values.openvas.tag }}" @@ -117,6 +110,7 @@ spec: mountPath: /mnt/ovc command: ['sh', '-c'] args: ["touch /mnt/ovc/openvas.log"] + {{ end }} - name: mqtt-broker-openvas-fix image: "{{ .Values.ospd.repository }}:{{ .Values.ospd.tag }}" imagePullPolicy: Always @@ -129,8 +123,6 @@ spec: {{ else }} args: ["sed 's/mqtt_server_uri = .*/openvasd_server = http:\\/\\/localhost:80/' /etc/openvas/openvas.conf > /mnt/ovc/openvas.conf; cp /etc/openvas/openvas_log.conf /mnt/ovc/"] {{ end }} - - {{ end }} containers: - name: redis image: "{{ .Values.redis.repository }}:{{ .Values.redis.tag }}" diff --git a/charts/openvasd/values.yaml b/charts/openvasd/values.yaml index 4cabd0e6b..d46f95660 100644 --- a/charts/openvasd/values.yaml +++ b/charts/openvasd/values.yaml @@ -42,35 +42,24 @@ openvasd: deploy_server: true deploy_client: true -# Required for version checks -notus: - repository: registry.community.greenbone.net/community/notus-scanner - pullPolicy: Always - tag: "edge" # NASL scripts also known as feed vulnerabilitytests: # latest is the most current community feed. repository: registry.community.greenbone.net/community/vulnerability-tests pullPolicy: Always - tag: "latest" + tag: "community" # Notus description json also known as feed notusdata: # latest is the most current community feed. repository: registry.community.greenbone.net/community/notus-data pullPolicy: Always - tag: "latest" - -# Required by notus -mqtt: - repository: registry.community.greenbone.net/community/mqtt-broker - pullPolicy: Always - tag: "latest" + tag: "community" # required by openvas and ospd redis: - repository: registry.community.greenbone.net/community/redis-server + repository: greenbone/redis-server pullPolicy: Always tag: "latest" From 0ea957886806a766bff67a5825a92653746f5f6a Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 14 Feb 2025 09:25:20 +0100 Subject: [PATCH 06/10] Fixes certificate dir of smoketest --- .github/workflows/functional.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 938566c59..5cb3bf83c 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -98,10 +98,10 @@ jobs: working-directory: rust/crates/smoketest env: SCAN_CONFIG: configs/simple_scan_ssh_only.json - CLIENT_KEY: ../../../examples/tls/self-signed/client.rsa - CLIENT_CERT: ../../../examples/tls/self-signed/client.pem + CLIENT_KEY: ../../examples/tls/self-signed/client.rsa + CLIENT_CERT: ../../examples/tls/self-signed/client.pem run: | - make build run + make build run || ls -las ../../ # - uses: actions/cache@v4 # with: # path: /tmp/openvas.tar From b0c6a9ea161f59f60052f2367897fd1d75ca58d1 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 14 Feb 2025 10:27:19 +0100 Subject: [PATCH 07/10] Adds kubectl commands after deployment for debug --- .github/workflows/functional.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 5cb3bf83c..1b3eb1dd3 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -73,11 +73,20 @@ jobs: docker image ls -a - name: 'do not, I repeat, do NOT pull that' run: | + # repository: registry.community.greenbone.net/community/openvas-scanner + # pullPolicy: Always + # tag: "edge" echo "openvas:" >> st.yaml + echo " repository: registry.community.greenbone.net/community/openvas-scanner" >> st.yaml + echo " tag: edge" >> st.yaml echo " pullPolicy: Never" >> st.yaml echo "vulnerabilitytests:" >> st.yaml + echo " repository: registry.community.greenbone.net/community/vulnerability-tests" >> st.yaml + echo " tag: community" >> st.yaml echo " pullPolicy: Never" >> st.yaml echo "notusdata:" >> st.yaml + echo " repository: registry.community.greenbone.net/community/notus-data" >> st.yaml + echo " tag: community" >> st.yaml echo " pullPolicy: Never" >> st.yaml cat st.yaml - name: deploy openvasd @@ -91,9 +100,10 @@ jobs: --values charts/openvasd/values.yaml \ --values charts/openvasd/mtls-wo-ingress.yaml \ --values st.yaml - kubectl rollout status --watch --timeout 600s deployment/openvasd --namespace openvasd echo "OPENVASD_SERVER=https://$(kubectl get svc -n openvasd | awk 'FNR == 2 {print $(3)}')" >> $GITHUB_ENV + - run: kubectl get pods -n openvasd -o=wide + - run: kubectl describe pod openvasd -n openvasd - name: smoketest working-directory: rust/crates/smoketest env: From 0134ba43474515f65350d83519105e77997c9abb Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 14 Feb 2025 10:56:50 +0100 Subject: [PATCH 08/10] Refactor use setup-rust instead of repeating cmds --- .github/workflows/tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0412103ba..d588e33f8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,9 +22,7 @@ jobs: working-directory: rust steps: - uses: actions/checkout@v4 - - run: sudo apt update || true - - run: sudo apt-get install -y libpcap-dev - - run: rustup update stable && rustup default stable || rustup default stable + - uses: ./.github/actions/setup-rust - name: unit-tests run: cargo test --lib --tests --workspace - name: experimental unit-tests From ea88108d96d0ded841137812d9fcb40d5cda5908 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 14 Feb 2025 11:06:27 +0100 Subject: [PATCH 09/10] Partially using ubuntu-latest for rust builds Until the self-hosted-generic has rustup installed ubuntu-latest is used for those tasks. --- .github/workflows/build.yml | 2 +- .github/workflows/functional.yaml | 2 +- .github/workflows/linting.yml | 8 ++++---- .github/workflows/tests.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 02a5e4cd3..8876422d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer cmake --build build Rust: - runs-on: "self-hosted-generic" + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-rust diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 1b3eb1dd3..c0b4a6d5a 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -23,7 +23,7 @@ jobs: - run: docker run --rm test ldd /usr/local/sbin/openvas | grep libopenvas_wmiclient - run: docker rmi test || true build-rs: - runs-on: self-hosted-generic + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: ./.github/actions/compile-x86_64 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index cf155b7df..a3e74a129 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -11,7 +11,7 @@ jobs: - name: Formatting run: sh .github/check-c-formatting.sh ${{ github.base_ref }} Rust: - runs-on: self-hosted-generic + runs-on: ubuntu-latest defaults: run: working-directory: rust @@ -21,7 +21,7 @@ jobs: - run: cargo clippy -- -D warnings - run: cargo fmt --check Rust-Typos: - runs-on: self-hosted-generic + runs-on: ubuntu-latest defaults: run: working-directory: rust @@ -31,7 +31,7 @@ jobs: - run: cargo install typos-cli || true - run: typos Rust-Audit: - runs-on: self-hosted-generic + runs-on: ubuntu-latest defaults: run: working-directory: rust @@ -40,7 +40,7 @@ jobs: - run: cargo install cargo-audit || true - run: cargo audit License-Headers: - runs-on: self-hosted-generic + runs-on: ubuntu-latest steps: - name: Check out openvas-scanner uses: actions/checkout@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d588e33f8..006cf5ca6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: cmake -Bbuild -DCMAKE_BUILD_TYPE=Release CTEST_OUTPUT_ON_FAILURE=1 cmake --build build -- tests test Rust: - runs-on: self-hosted-generic + runs-on: ubuntu-latest defaults: run: working-directory: rust From 1e9c3897420b2d64d16837e7b37c8bbf675685db Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 14 Feb 2025 11:11:42 +0100 Subject: [PATCH 10/10] Remove smoketests as it is moved to functional --- .github/workflows/control.yml | 11 ++----- .github/workflows/smoketest.yaml | 51 -------------------------------- 2 files changed, 2 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/smoketest.yaml diff --git a/.github/workflows/control.yml b/.github/workflows/control.yml index 4f5861a83..a8479cfe0 100644 --- a/.github/workflows/control.yml +++ b/.github/workflows/control.yml @@ -143,18 +143,11 @@ jobs: email: ${{ secrets.GREENBONE_BOT_MAIL }} gpg_key: ${{ secrets.GPG_KEY }} gpg_pass: ${{ secrets.GPG_PASSPHRASE }} - smoketests: - if: github.event_name != 'pull_request' - needs: [container, init] - uses: ./.github/workflows/smoketest.yaml - with: - docker_repo: ${{ github.repository }} - docker_tag: ${{needs.init.outputs.docker_test_tag}} helm: + if: github.event_name != 'pull_request' && vars.IMAGE_REGISTRY != '' + needs: [container, init] permissions: packages: write - needs: [smoketests] - if: vars.IMAGE_REGISTRY != '' uses: ./.github/workflows/push-helm-chart.yml secrets: user: ${{ secrets.GREENBONE_BOT }} diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml deleted file mode 100644 index 66cff980f..000000000 --- a/.github/workflows/smoketest.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: Smoketests - -on: - workflow_call: - inputs: - docker_repo: - type: string - default: "greenbone/openvas-scanner" - docker_tag: - required: true - type: string - -# smoke test definition. -# It depends on build.yml that is controlled via control.yml -# -jobs: - openvasd: - runs-on: self-hosted-generic - steps: - - uses: actions/checkout@v4 - - name: 'Set openvas-scanner-image' - run: | - echo "openvas:" >> st.yaml - echo " repository: ${{ inputs.docker_repo }}" >> st.yaml - echo " pullPolicy: Always" >> st.yaml - echo " tag: \"${{ inputs.docker_tag }}\"" >> st.yaml - cat st.yaml - - name: Start a local k8s cluster - uses: jupyterhub/action-k3s-helm@v4 - with: - k3s-channel: latest - metrics-enabled: false - - name: deploy openvasd - run: | - cd rust/examples/tls/self-signed - make delete deploy - cd - - - helm uninstall openvasd --namespace openvasd|| true - helm install --namespace openvasd --create-namespace openvasd charts/openvasd/ --values charts/openvasd/values.yaml --values charts/openvasd/mtls-wo-ingress.yaml --values st.yaml - - kubectl rollout status --watch --timeout 600s deployment/openvasd --namespace openvasd - echo "OPENVASD_SERVER=https://$(kubectl get svc -n openvasd | awk 'FNR == 2 {print $(3)}')" >> $GITHUB_ENV - - name: smoketest - working-directory: rust/crates/smoketest - env: - SCAN_CONFIG: configs/simple_scan_ssh_only.json - CLIENT_KEY: ../examples/tls/self-signed/client.rsa - CLIENT_CERT: ../examples/tls/self-signed/client.pem - run: | - make build run