Merge pull request #144 from inab/143-integrate-pip-audit-in-the-ci-w…

…orkflows

First test integrating 143 integrate pip audit in the ci workflows

.github/workflows/pip-audit.yml

@@ -0,0 +1,75 @@
+name: pip-audit
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * 1"
+
+jobs:
+  pip-audit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
+    name: pip-audit python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: |
+            requirements.txt
+          architecture: x64
+      - name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
+        run: |
+          pip install --upgrade pip wheel
+          pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
+#      - name: 'Freeze Python ${{ matrix.python-version }} constraints'
+#        run: |
+#          pip freeze > constraints-${{ matrix.python-version }}.txt
+      - id: gen-cve-output
+        uses: pypa/[email protected]
+      - name: show_markdown
+        run: |
+          # echo "storing to file: ${{ steps.gen-cve-output.outputs.internal-be-careful-output }}"
+          # echo ${{ steps.gen-cve-output.outputs.internal-be-careful-output }} > $GITHUB_WORKSPACE/security_scans.md
+          # echo "saved."
+          type -a pip-audit
+          cat <<EOF
+          ${{ steps.gen-cve-output.outputs.internal-be-careful-output }}
+          EOF
+#      - uses: actions/upload-artifact@v3
+#        with:
+#          retention-days: 2
+#          path: constraints-${{ matrix.python-version }}.txt
+#
+#  pull_request_changes:
+#    # Do this only when it is not a pull request validation
+#    if: github.event_name != 'pull_request'
+#    runs-on: ubuntu-latest
+#    name: Pull request with the newly generated contents
+#    needs:
+#      - pre-commit
+#    steps:
+#      - uses: actions/checkout@v3
+#      - uses: actions/download-artifact@v3
+#        with:
+#          path: changes-dir
+#      - name: Move artifacts to their right place
+#        run: |
+#          cp -dpr changes-dir/artifact/* .
+#          rm -r changes-dir/artifact
+#      - name: Create Pull Request
+#        id: cpr
+#        uses: peter-evans/create-pull-request@v5
+#        with:
+#          title: Updated constraints (triggered by ${{ github.sha }})
+#          branch: create-pull-request/patch-constraints
+#          delete-branch: true
+#          commit-message: "[create-pull-request] Automatically commit updated contents (constraints)"
+#      - name: Check outputs
+#        if: ${{ steps.cpr.outputs.pull-request-number }}
+#        run: |
+#          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+#          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.github/workflows/pre-commit.yml

@@ -34,23 +34,18 @@ jobs:
           files: requirements.txt
 
       - name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
-        if: ${{ matrix.python-version != '3.6' }}
         run: |
           pip install --upgrade pip wheel
           if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then
             pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
           else
             pip install -r requirements.txt
           fi
-      #- name: 'Install requirements  (custom Python ${{ matrix.python-version }})'
-      #  if: ${{ matrix.python-version == '3.6' }}
-      #  run: |
-      #    pip install wheel
-      #    pip install -r requirements.txt -r requirements-additional-${{ matrix.python-version }}.txt
       - name: 'Freeze Python ${{ matrix.python-version }} constraints'
         run: |
           pip freeze > constraints-${{ matrix.python-version }}.txt
-      - run: pip install -r dev-requirements.txt -r mypy-requirements.txt
+      - run: |
+          pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt
       - name: MyPy cache
         uses: actions/cache@v3
         with:

.pre-commit-config.yaml

@@ -42,6 +42,15 @@ repos:
         require_serial: true
         entry: mypy
         args: [--strict, --show-error-codes, --no-warn-unused-ignores]
+      - id: pip-audit-local
+        name: pip-audit over local environment
+        stages: [manual]
+        language: system
+        always_run: true
+        pass_filenames: false
+        require_serial: true
+        entry: pip-audit
+        args: []
 
 ## Main problem: python executable path, used to find the environment, is hardcoded
 #  - repo: https://github.com/pre-commit/mirrors-mypy.git
@@ -106,3 +115,9 @@ repos:
     rev: v0.1.0
     hooks:
       - id: cff_ver_validate
+  - repo: https://github.com/pypa/pip-audit
+    rev: v2.7.3
+    hooks:
+      - id: pip-audit
+        stages: [manual]
+        args: ["-r", "requirements.txt"]

INSTALL.md

@@ -256,7 +256,8 @@ All the development dependencies are declared at [dev-requirements.txt](dev-requ
 python3 -m venv .pyWEenv
 source .pyWEenv/bin/activate
 pip install --require-virtualenv --upgrade pip wheel
-pip install --require-virtualenv -r requirements.txt -r dev-requirements.txt -r mypy-requirements.txt
+pyver="$(python -c 'import sys;print(f"{sys.version_info[0]}.{sys.version_info[1]}")')"
+pip install --require-virtualenv -r requirements.txt -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${pyver}.txt
 ```
 
 One of these dependencies is [pre-commit](https://pre-commit.com/), whose rules are declared at [.pre-commit-config.yaml](.pre-commit-config.yaml) (there are special versions of these rules for GitHub).

container_recipes/basic-installer.bash

@@ -259,11 +259,12 @@ if [ -z "$envDir" ]; then
 	if [ -f "$constraintsFile" ] ; then
 		PIP_INSTALL_PARAMS+=( -c "${constraintsFile}" )
 	fi
-	pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}"
 
-	# Now, should we run something wrapped?
-	if [ $# != 0 ] ; then
-		pip install --require-virtualenv -r "${wfexsDir}"/dev-requirements.txt -r "${wfexsDir}"/mypy-requirements.txt
+	# Now, should we run something wrapped (for development purposes)?
+	if [ $# = 0 ] ; then
+		pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}"
+	else
+		pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}" -r "${wfexsDir}"/dev-requirements.txt -r "${wfexsDir}"/mypy-requirements.txt
 		"$@"
 	fi
 fi

dev-requirements.txt

@@ -18,3 +18,6 @@ mypy >= 1.1.1
 pre-commit >= 2.17.0
 # This is the last version of black supporting Python 3.7
 black == 23.3.0
+# pip-audit, depending on the minimal versions
+pip-audit < 2.6.2 ; python_version == '3.7'
+pip-audit >= 2.6.2 ; python_version >= '3.8'