From 543ff90ae3c120abff450d19d6a500bad576a031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=C2=AA=20Fern=C3=A1ndez?= Date: Thu, 16 Jan 2025 12:58:07 +0100 Subject: [PATCH 1/6] Integrated `pip-audit` (in two different flavors) as manual steps in pre-commit. --- .pre-commit-config.yaml | 14 ++++++++++++++ dev-requirements.txt | 3 +++ 2 files changed, 17 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc75ce2a..2642a58d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,15 @@ repos: require_serial: true entry: mypy args: [--strict, --show-error-codes, --no-warn-unused-ignores] + - id: pip-audit-local + name: pip-audit over local environment + stages: [manual] + language: system + always_run: true + pass_filenames: false + require_serial: true + entry: pip-audit + args: [] ## Main problem: python executable path, used to find the environment, is hardcoded # - repo: https://github.com/pre-commit/mirrors-mypy.git @@ -106,3 +115,8 @@ repos: rev: v0.1.0 hooks: - id: cff_ver_validate + - repo: https://github.com/pypa/pip-audit + rev: v2.7.3 + hooks: + - id: pip-audit + args: ["-r", "requirements.txt"] \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index b8bb3f05..3e8888a3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -18,3 +18,6 @@ mypy >= 1.1.1 pre-commit >= 2.17.0 # This is the last version of black supporting Python 3.7 black == 23.3.0 +# pip-audit, depending on the minimal versions +pip-audit < 2.6.2 ; python_version == '3.7' +pip-audit >= 2.6.2 ; python_version >= '3.8' \ No newline at end of file From 38ff88254a6a2b28e4c182cb2a0d999da2f23629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=C2=AA=20Fern=C3=A1ndez?= Date: Thu, 16 Jan 2025 13:00:39 +0100 Subject: [PATCH 2/6] Installation of development dependencies now uses existing constraints Installation of development dependencies should respect the fixed versions already recorded at the corresponding constraints file. So, use it whenever it is possible. --- .github/workflows/pre-commit.yml | 9 ++------- INSTALL.md | 3 ++- container_recipes/basic-installer.bash | 9 +++++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 303c317d..144980bc 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -34,7 +34,6 @@ jobs: files: requirements.txt - name: 'Install requirements (standard or constraints ${{ matrix.python-version }})' - if: ${{ matrix.python-version != '3.6' }} run: | pip install --upgrade pip wheel if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then @@ -42,15 +41,11 @@ jobs: else pip install -r requirements.txt fi - #- name: 'Install requirements (custom Python ${{ matrix.python-version }})' - # if: ${{ matrix.python-version == '3.6' }} - # run: | - # pip install wheel - # pip install -r requirements.txt -r requirements-additional-${{ matrix.python-version }}.txt - name: 'Freeze Python ${{ matrix.python-version }} constraints' run: | pip freeze > constraints-${{ matrix.python-version }}.txt - - run: pip install -r dev-requirements.txt -r mypy-requirements.txt + - run: | + pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt - name: MyPy cache uses: actions/cache@v3 with: diff --git a/INSTALL.md b/INSTALL.md index 18e4afb9..b4473621 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -256,7 +256,8 @@ All the development dependencies are declared at [dev-requirements.txt](dev-requ python3 -m venv .pyWEenv source .pyWEenv/bin/activate pip install --require-virtualenv --upgrade pip wheel -pip install --require-virtualenv -r requirements.txt -r dev-requirements.txt -r mypy-requirements.txt +pyver="$(python -c 'import sys;print(f"{sys.version_info[0]}.{sys.version_info[1]}")')" +pip install --require-virtualenv -r requirements.txt -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${pyver}.txt ``` One of these dependencies is [pre-commit](https://pre-commit.com/), whose rules are declared at [.pre-commit-config.yaml](.pre-commit-config.yaml) (there are special versions of these rules for GitHub). diff --git a/container_recipes/basic-installer.bash b/container_recipes/basic-installer.bash index a808b54b..a9db1194 100755 --- a/container_recipes/basic-installer.bash +++ b/container_recipes/basic-installer.bash @@ -259,11 +259,12 @@ if [ -z "$envDir" ]; then if [ -f "$constraintsFile" ] ; then PIP_INSTALL_PARAMS+=( -c "${constraintsFile}" ) fi - pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}" - # Now, should we run something wrapped? - if [ $# != 0 ] ; then - pip install --require-virtualenv -r "${wfexsDir}"/dev-requirements.txt -r "${wfexsDir}"/mypy-requirements.txt + # Now, should we run something wrapped (for development purposes)? + if [ $# = 0 ] ; then + pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}" + else + pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}" -r "${wfexsDir}"/dev-requirements.txt -r "${wfexsDir}"/mypy-requirements.txt "$@" fi fi From ba739d0fd4906af3c76a29015f293349dd9e04da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=C2=AA=20Fern=C3=A1ndez?= Date: Thu, 16 Jan 2025 13:48:33 +0100 Subject: [PATCH 3/6] Forgot to label as manual the pip-audit pre-commit step --- .pre-commit-config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2642a58d..84d02686 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -118,5 +118,6 @@ repos: - repo: https://github.com/pypa/pip-audit rev: v2.7.3 hooks: - - id: pip-audit - args: ["-r", "requirements.txt"] \ No newline at end of file + - id: pip-audit + stages: [manual] + args: ["-r", "requirements.txt"] \ No newline at end of file From 1b7c031609dd081a4b8a4abab4b3fc342fa5ce17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=C2=AA=20Fern=C3=A1ndez?= Date: Thu, 16 Jan 2025 13:49:41 +0100 Subject: [PATCH 4/6] Added prototype GitHub CI workflow to audit python dependencies --- .github/workflows/pip-audit.yml | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/pip-audit.yml diff --git a/.github/workflows/pip-audit.yml b/.github/workflows/pip-audit.yml new file mode 100644 index 00000000..b30062b0 --- /dev/null +++ b/.github/workflows/pip-audit.yml @@ -0,0 +1,65 @@ +name: pip-audit + +on: + workflow_dispatch: + schedule: + - cron: "0 12 * * 1" + +jobs: + pip-audit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ] + name: pip-audit python ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + requirements.txt + architecture: x64 + - name: 'Install requirements (standard or constraints ${{ matrix.python-version }})' + run: | + pip install --upgrade pip wheel + pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt +# - name: 'Freeze Python ${{ matrix.python-version }} constraints' +# run: | +# pip freeze > constraints-${{ matrix.python-version }}.txt + - uses: pypa/gh-action-pip-audit@v1.1.0 +# - uses: actions/upload-artifact@v3 +# with: +# retention-days: 2 +# path: constraints-${{ matrix.python-version }}.txt +# +# pull_request_changes: +# # Do this only when it is not a pull request validation +# if: github.event_name != 'pull_request' +# runs-on: ubuntu-latest +# name: Pull request with the newly generated contents +# needs: +# - pre-commit +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/download-artifact@v3 +# with: +# path: changes-dir +# - name: Move artifacts to their right place +# run: | +# cp -dpr changes-dir/artifact/* . +# rm -r changes-dir/artifact +# - name: Create Pull Request +# id: cpr +# uses: peter-evans/create-pull-request@v5 +# with: +# title: Updated constraints (triggered by ${{ github.sha }}) +# branch: create-pull-request/patch-constraints +# delete-branch: true +# commit-message: "[create-pull-request] Automatically commit updated contents (constraints)" +# - name: Check outputs +# if: ${{ steps.cpr.outputs.pull-request-number }} +# run: | +# echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" +# echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" From 8235e8aa549094f27fa72dfa4d1458b9c5fc1fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=C2=AA=20Fern=C3=A1ndez?= Date: Fri, 17 Jan 2025 13:28:00 +0100 Subject: [PATCH 5/6] Added a method to print the report meanwhile is being developed --- .github/workflows/pip-audit.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pip-audit.yml b/.github/workflows/pip-audit.yml index b30062b0..12861af4 100644 --- a/.github/workflows/pip-audit.yml +++ b/.github/workflows/pip-audit.yml @@ -28,7 +28,16 @@ jobs: # - name: 'Freeze Python ${{ matrix.python-version }} constraints' # run: | # pip freeze > constraints-${{ matrix.python-version }}.txt - - uses: pypa/gh-action-pip-audit@v1.1.0 + - id: gen-cve-output + uses: pypa/gh-action-pip-audit@v1.1.0 + - name: show_markdown + run: | + # echo "storing to file: ${{ steps.gen-cve-output.outputs.internal-be-careful-output }}" + # echo ${{ steps.gen-cve-output.outputs.internal-be-careful-output }} > $GITHUB_WORKSPACE/security_scans.md + # echo "saved." + cat < Date: Fri, 17 Jan 2025 13:29:01 +0100 Subject: [PATCH 6/6] Another hint to understand what it is available after running the previous action --- .github/workflows/pip-audit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pip-audit.yml b/.github/workflows/pip-audit.yml index 12861af4..d9aa4e5c 100644 --- a/.github/workflows/pip-audit.yml +++ b/.github/workflows/pip-audit.yml @@ -35,6 +35,7 @@ jobs: # echo "storing to file: ${{ steps.gen-cve-output.outputs.internal-be-careful-output }}" # echo ${{ steps.gen-cve-output.outputs.internal-be-careful-output }} > $GITHUB_WORKSPACE/security_scans.md # echo "saved." + type -a pip-audit cat <