From d423a0abae25f80ff343f90fc9831103cc311b13 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Wed, 12 Jul 2023 11:13:28 +0200 Subject: [PATCH] common: test coverity scan --- .github/workflows/centos.yml | 45 ------------------ .github/workflows/docker_rebuild.yml | 55 ---------------------- .github/workflows/main.yml | 53 --------------------- .github/workflows/nightly.yml | 60 ------------------------ .github/workflows/pmem_check.yml | 41 ----------------- .github/workflows/pmem_long.yml | 41 ----------------- .github/workflows/pmem_ras.yml | 66 --------------------------- .github/workflows/pmem_tests_main.yml | 16 ------- .github/workflows/pmem_valgrind.yml | 42 ----------------- .github/workflows/scan_coverity.yml | 2 - .github/workflows/scans.yml | 8 ++-- .github/workflows/ubuntu.yml | 51 --------------------- 12 files changed, 5 insertions(+), 475 deletions(-) delete mode 100644 .github/workflows/centos.yml delete mode 100644 .github/workflows/docker_rebuild.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/nightly.yml delete mode 100644 .github/workflows/pmem_check.yml delete mode 100644 .github/workflows/pmem_long.yml delete mode 100644 .github/workflows/pmem_ras.yml delete mode 100644 .github/workflows/pmem_tests_main.yml delete mode 100644 .github/workflows/pmem_valgrind.yml delete mode 100644 .github/workflows/ubuntu.yml diff --git a/.github/workflows/centos.yml b/.github/workflows/centos.yml deleted file mode 100644 index c6720cfe139..00000000000 --- a/.github/workflows/centos.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Validate on CentoOS. -# CentOS 7 is maintain in PMDK project as it is required for DAOS. -name: CentOS - -on: - workflow_dispatch: - workflow_call: - -jobs: - linux: - name: CentOS - runs-on: ubuntu-latest - env: - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - HOST_WORKDIR: /home/runner/work/pmdk/pmdk - WORKDIR: utils/docker - PMDK_CC: gcc - PMDK_CXX: g++ - MAKE_PKG: 0 - VALGRIND: 1 - SRC_CHECKERS: 0 - OS: centos - strategy: - matrix: - CONFIG: [ - "OS_VER=7 FAULT_INJECTION=1 TEST_BUILD=debug", - "OS_VER=7 TEST_BUILD=nondebug", - "OS_VER=7 TEST_BUILD=debug NDCTL_ENABLE=n", - "OS_VER=7 TEST_BUILD=nondebug NDCTL_ENABLE=n", - "OS_VER=stream TEST_BUILD=debug", - "OS_VER=stream FAULT_INJECTION=1 TEST_BUILD=debug", - "OS_VER=stream TEST_BUILD=nondebug", - ] - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - fetch-depth: 50 - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh - - - name: Run the build - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh diff --git a/.github/workflows/docker_rebuild.yml b/.github/workflows/docker_rebuild.yml deleted file mode 100644 index fdb6b16a5b6..00000000000 --- a/.github/workflows/docker_rebuild.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Rebuild docker images if needed. -name: Docker rebuild - -on: - workflow_dispatch: - pull_request: - branches: - - master - paths: - - 'utils/docker/images/**' - push: - branches: - - master - paths: - - 'utils/docker/images/**' - schedule: - # run this job at 22:00 UTC every other day (hopefully just before the nightly) - - cron: '0 22 */2 * *' - -env: - # use org's Private Access Token to log in to GitHub Container Registry - GH_CR_USER: ${{ secrets.GH_CR_USER }} - GH_CR_PAT: ${{ secrets.GH_CR_PAT }} - - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - WORKDIR: utils/docker - PUSH_IMAGE: 1 - -jobs: - image: - if: github.repository == 'pmem/pmdk' - name: Image - runs-on: ubuntu-latest - strategy: - matrix: - CONFIG: [ - "OS=centos OS_VER=7", - "OS=centos OS_VER=stream", - "OS=debian OS_VER=11", - "OS=fedora OS_VER=37", - "OS=opensuse-leap OS_VER=15", - "OS=rockylinux OS_VER=8", - "OS=rockylinux OS_VER=9", - "OS=ubuntu OS_VER=22.04", - ] - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - - - name: Get system information - run: ./$WORKDIR/get-system-info.sh - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 01ecb83f76f..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Validate on default OSes. -name: Main - -on: - workflow_dispatch: - pull_request: - -jobs: - src_checkers: - name: Source checkers - runs-on: ubuntu-latest - env: - NDCTL_ENABLE: n - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - - - name: Install dependencies - run: sudo pip install flake8 - - - name: Check licenses - run: make -j$(nproc) check-license - - - name: Check style - run: make -j$(nproc) cstyle - - - basic_build: - name: Basic build - runs-on: ubuntu-latest - env: - NDCTL_ENABLE: n - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - - - name: Install dependencies - run: sudo apt-get -y install pandoc - - - name: Build sources - run: make -j$(nproc) test - - - call-ubuntu: - needs: [src_checkers, basic_build] - uses: ./.github/workflows/ubuntu.yml - name: Ubuntu - - - call-centos: - needs: [src_checkers, basic_build] - uses: ./.github/workflows/centos.yml - name: CentOS diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index fa499eed45d..00000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Run less frequently used OSes and checks. -name: Nightly - -on: - workflow_dispatch: - schedule: - # run this job at 23:00 UTC every other day - - cron: '0 23 */2 * *' - push: - tags: - - '*' - -env: - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - HOST_WORKDIR: /home/runner/work/pmdk/pmdk - WORKDIR: utils/docker - PMDK_CC: gcc - PMDK_CXX: g++ - MAKE_PKG: 0 - VALGRIND: 1 - SRC_CHECKERS: 0 - -jobs: - linux: - name: Linux - runs-on: ubuntu-latest - strategy: - matrix: - CONFIG: [ - "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n", - "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", - "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", - "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=nondebug", - "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=debug", - "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug", - "OS=opensuse-leap OS_VER=15 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0", - "OS=rockylinux OS_VER=8 TEST_BUILD=debug", - "OS=rockylinux OS_VER=8 TEST_BUILD=nondebug", - "OS=rockylinux OS_VER=8 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0", - "OS=rockylinux OS_VER=9 TEST_BUILD=debug", - "OS=rockylinux OS_VER=9 TEST_BUILD=nondebug", - "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0", - "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug", - "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", - ] - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - fetch-depth: 50 - - - name: Get system information - run: ./$WORKDIR/get-system-info.sh - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild - - - name: Run the build - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh diff --git a/.github/workflows/pmem_check.yml b/.github/workflows/pmem_check.yml deleted file mode 100644 index 67627dc0c3f..00000000000 --- a/.github/workflows/pmem_check.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Run short and medium tests (as defined in test frameworks), equivalent to 'make check'. -# -# This workflow is run on 'self-hosted' runners. -name: PMEM check - -on: - workflow_dispatch: - workflow_call: - -jobs: - linux: - name: PMEM_check - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [[self-hosted, rhel],[self-hosted, opensuse]] - build: ['debug', 'nondebug', 'static_debug', 'static_nondebug'] - env: - WORKDIR: utils/gha-runners - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - repository: pmem/pmdk - - - name: Get system information - run: ./$WORKDIR/get-system-info.sh - - - name: Build - run: ./$WORKDIR/build-pmdk.sh - - - name: Create testconfig files - run: ./$WORKDIR/create-testconfig.sh - - - name: Run tests (Bash) - run: cd src/test/ && ./RUNTESTS.sh -b ${{ matrix.build }} - - - name: Run tests (Python) - run: cd src/test/ && ./RUNTESTS.py -b ${{ matrix.build }} diff --git a/.github/workflows/pmem_long.yml b/.github/workflows/pmem_long.yml deleted file mode 100644 index 6324501d182..00000000000 --- a/.github/workflows/pmem_long.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Run tests of type 'long' (as defined in test frameworks). -# -# This workflow is run on 'self-hosted' runners. -name: PMEM long - -on: - workflow_dispatch: - schedule: - - cron: '5 0 * * 6' # At 00:05 on Saturday. - -jobs: - linux: - name: PMEM_long - if: github.repository == 'pmem/pmdk' - runs-on: ${{ matrix.os }} - timeout-minutes: 960 - strategy: - fail-fast: false - matrix: - os: [[self-hosted, rhel],[self-hosted, opensuse]] - script: ['sh', 'py'] - build: [ 'debug', 'nondebug', 'static_debug', 'static_nondebug'] - - env: - WORKDIR: utils/gha-runners - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - - - name: Get system information - run: ./$WORKDIR/get-system-info.sh - - - name: Build - run: ./$WORKDIR/build-pmdk.sh - - - name: Create testconfig file - run: ./$WORKDIR/create-testconfig.sh - - - name: Run tests - run: cd src/test/ && ./RUNTESTS.${{ matrix.script }} -t long -b ${{ matrix.build }} diff --git a/.github/workflows/pmem_ras.yml b/.github/workflows/pmem_ras.yml deleted file mode 100644 index 703f48278b9..00000000000 --- a/.github/workflows/pmem_ras.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Run RAS test: Unsafe Shutdown Local. -# -# This workflow is run on 'self-hosted' runners. -# -# RAS tests require a different approach compared to the standard tests - they need to -# reboot the runner during the test. Normally, rebooting and continuing the job on GHA -# is not possible, due to losing connection with the runner. To work around this issue, -# an additional runner (not connected to the GH) runs the tests instead. -# -# The general idea of the solution is: -# - First platform [self-hosted runner] functions as the controller [ras_controller], -# - Second platform functions as the test runner [ras_runner], -# - The workflow launches its steps on the controller, -# - The controller will then run an ansible playbook on the second platform [ras_runner], -# with options provided by the workflow, -# - The test runner follows the steps given by the controller, -# running the tests in the process and providing results as output, -# - The controller gathers this output and prints it in GHA job. -# -# The only drawback of this idea is that workflow would always finish successfully. -# The solution was added as an additional step, at the end of the workflow, parsing the output. -# -# More detailed information about the ansible playbook and tests themselves can be found in: -# utils/gha-runners/run-ras-linux.yml -name: PMEM RAS - -on: - workflow_dispatch: - schedule: - # run this job every 8 hours - - cron: '0 */8 * * *' - -jobs: - linux: - name: PMEM_RAS - if: github.repository == 'pmem/pmdk' - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [[self-hosted, ras_controller]] - env: - WORKDIR: utils/gha-runners - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - - # Variables, such as $ras_runner are hidden on the controller platform as environmental variables. - # 'sed' command is used to filter out IP addresses from the ansible output, it will show up as the 'ras_runner' instead. - # 'tee' command is used to save the overall output to the file. This file is needed for the next step. - - name: Prepare and run RAS Linux tests via ansible-playbook - run: | - cd $WORKDIR - ansible-playbook -i $ras_runner, run-ras-linux.yml -e "host=all ansible_user=$ras_user" | sed 's/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/ras_runner/' | tee playbook_output.txt - - # This simple step will look through the output in search of specific fail strings. - # If any phrase is found in the file, the workflow will fail. - - name: Fail the workflow if the playbook finished with a failure - run: | - cd $WORKDIR - if grep -E 'fatal: \[ras_runner\]: FAILED!|failed: \[ras_runner\]' "playbook_output.txt"; then - exit 1 - else - exit 0 - fi diff --git a/.github/workflows/pmem_tests_main.yml b/.github/workflows/pmem_tests_main.yml deleted file mode 100644 index a0ae1981d2c..00000000000 --- a/.github/workflows/pmem_tests_main.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Run all PMem tests in a sequence -# -# This workflow is run on 'self-hosted' runners. -name: PMEM tests main - -on: - workflow_dispatch: - schedule: - # run this job every 8 hours - - cron: '0 */8 * * *' - -jobs: - call-pmem-check: - uses: ./.github/workflows/pmem_check.yml - name: PMem check - if: github.repository == 'pmem/pmdk' diff --git a/.github/workflows/pmem_valgrind.yml b/.github/workflows/pmem_valgrind.yml deleted file mode 100644 index 87ddb6d6183..00000000000 --- a/.github/workflows/pmem_valgrind.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Run all tests with 'force-enabled' valgrind. -# -# This workflow is run on 'self-hosted' runners. -name: PMEM Valgrind - -on: - workflow_dispatch: - schedule: - - cron: '0 23 * * 6' # At 23:00 on Saturday. - -jobs: - linux: - name: PMEM_valgrind - if: github.repository == 'pmem/pmdk' - runs-on: ${{ matrix.os }} - timeout-minutes: 4320 # wait max 3 days - strategy: - fail-fast: false - matrix: - script: ['sh', 'py'] - config: ['drd', 'pmemcheck', 'memcheck', 'helgrind'] - build: ['debug', 'nondebug', 'static_debug', 'static_nondebug'] - os: [[self-hosted, rhel],[self-hosted, opensuse]] - - env: - WORKDIR: utils/gha-runners - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - - - name: Get system information - run: ./$WORKDIR/get-system-info.sh - - - name: Build - run: ./$WORKDIR/build-pmdk.sh - - - name: Create testconfig file - run: ./$WORKDIR/create-testconfig.sh - - - name: Run tests - run: cd src/test/ && ./RUNTESTS.${{ matrix.script }} --force-enable ${{ matrix.config }} -b ${{ matrix.build }} diff --git a/.github/workflows/scan_coverity.yml b/.github/workflows/scan_coverity.yml index 21be9ec8954..9feda3fd0b7 100644 --- a/.github/workflows/scan_coverity.yml +++ b/.github/workflows/scan_coverity.yml @@ -8,8 +8,6 @@ on: env: GITHUB_REPO: pmem/pmdk DOCKER_REPO: ghcr.io/pmem/pmdk - COVERITY_SCAN_NOTIFICATION_EMAIL: ${{ secrets.COVERITY_SCAN_NOTIFICATION_EMAIL }} - COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} HOST_WORKDIR: /home/runner/work/pmdk/pmdk WORKDIR: utils/docker PMDK_CC: gcc diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml index 60cd71578f7..90721f7daab 100644 --- a/.github/workflows/scans.yml +++ b/.github/workflows/scans.yml @@ -3,9 +3,11 @@ name: Scans on: workflow_dispatch: - schedule: - # run this job at 00:00 UTC every day - - cron: '0 0 * * *' + pull_request: + +env: + COVERITY_SCAN_NOTIFICATION_EMAIL: ${{ secrets.COVERITY_SCAN_NOTIFICATION_EMAIL }} + COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} jobs: call-bandit: diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index 62bb6bb74b0..00000000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Validate on Ubuntu. -name: Ubuntu - -on: - workflow_dispatch: - workflow_call: - -env: - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - -jobs: - linux: - name: Linux - runs-on: ubuntu-latest - env: - # use org's Private Access Token to log in to GitHub Container Registry - DOC_UPDATE_GITHUB_TOKEN: ${{ secrets.DOC_UPDATE_GITHUB_TOKEN }} - HOST_WORKDIR: /home/runner/work/pmdk/pmdk - WORKDIR: utils/docker - PMDK_CC: gcc - PMDK_CXX: g++ - MAKE_PKG: 0 - VALGRIND: 1 - SRC_CHECKERS: 0 - strategy: - # Introduced to stop sporadically failing Codecov uploader (COVERAGE=1) - # from stopping all other builds. - fail-fast: false - matrix: - CONFIG: [ - "N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug", - "N=2 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=nondebug UBSAN=1", - "N=3 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug", - "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", - "N=5 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug", - ] - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - fetch-depth: 50 - - - name: Get system information - run: ./$WORKDIR/get-system-info.sh - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh - - - name: Run the build - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh