diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 2270f496..00000000 --- a/.dockerignore +++ /dev/null @@ -1,27 +0,0 @@ -# Docker related -development/Dockerfile -development/docker-compose*.yml -development/*.env -*.env -environments/ - -# Python -**/*.pyc -**/*.pyo -**/__pycache__/ -**/.pytest_cache/ -**/.venv/ - - -# Other -docs/_build -FAQ.md -.git/ -.gitignore -.github -tasks.py -LICENSE -**/*.log -**/.vscode/ -invoke*.yml -tasks.py diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md index 3950227c..674ff234 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -1,31 +1,20 @@ - - -# Closes: # +Optimize CI time. ## What's Changed - +- Added `build` GitHub action. +- Added simple `test-feature-pr.yml` GitHub workflow. +- Bumped CI `uses` versions to the latest major version each. +- Bumped CI image version to `ubuntu-22.04`. +- Disabled `ci.yml` GitHub workflow. +- Moved Compose dependencies to database `yaml` files. +- Added `--test-docs` to `invoke tests`. +- Fixed `invoke tests --lint-only` not to run coverage. ## To Do - - [ ] Explanation of Change(s) - [ ] Added change log fragment(s) (for more information see [the documentation](https://docs.nautobot.com/projects/core/en/stable/development/#creating-changelog-fragments)) - [ ] Attached Screenshots, Payload Example diff --git a/.github/actions/config-compose/action.yml b/.github/actions/config-compose/action.yml new file mode 100644 index 00000000..a177d33d --- /dev/null +++ b/.github/actions/config-compose/action.yml @@ -0,0 +1,146 @@ +--- +name: "Configure Compose" +description: "Configure Docker Compose for Nautobot App" +inputs: + db-backend: + description: "Database Backend" + required: false + default: "" + nautobot-version: + description: "Nautobot Version" + required: true + python-version: + description: "Python Version" + required: true + tag-prefix: + description: "Docker Image Tag Prefix" + required: true + use-cache: + description: "Use GitHub Actions Cache" + required: false + default: "false" +runs: + using: "composite" + steps: + - name: "Configure" + id: "config" + shell: "bash" + run: | + cd development + + cp creds.example.env creds.env + + GHCR_IMAGE_PREFIX="ghcr.io/${{ github.repository }}/nautobot-dev" + GHCR_IMAGE_TAG="${{ inputs.tag-prefix }}-${{ inputs.nautobot-version }}-py${{ inputs.python-version }}" + + export COMPOSE_ANSI=0 + + COMPOSE_FILE="docker-compose.base.yml" + if [[ -n "${{ inputs.db-backend }}" ]]; then + COMPOSE_FILE="$COMPOSE_FILE:docker-compose.${{ inputs.db-backend }}.yml:docker-compose.redis.yml" + fi + export COMPOSE_FILE="$COMPOSE_FILE:docker-compose.dev.yml" + + export NAUTOBOT_VER="${{ inputs.nautobot-version }}" + export PYTHON_VER="${{ inputs.python-version }}" + + COMPOSE_IMAGE="$(docker compose convert --format json | jq -r .services.nautobot.image)" + + echo "COMPOSE_FILE=$COMPOSE_FILE" | tee -a "$GITHUB_ENV" + echo "NAUTOBOT_VER=$NAUTOBOT_VER" | tee -a "$GITHUB_ENV" + echo "PYTHON_VER=$PYTHON_VER" | tee -a "$GITHUB_ENV" + + echo "compose-file=$COMPOSE_FILE" | tee -a "$GITHUB_OUTPUT" + echo "compose-image=$COMPOSE_IMAGE" | tee -a "$GITHUB_OUTPUT" + echo "ghcr-image=$GHCR_IMAGE_PREFIX:$GHCR_IMAGE_TAG" | tee -a "$GITHUB_OUTPUT" + echo "ghcr-image-prefix=$GHCR_IMAGE_PREFIX" | tee -a "$GITHUB_OUTPUT" + echo "ghcr-image-tag=$GHCR_IMAGE_TAG" | tee -a "$GITHUB_OUTPUT" + + if [[ -n "${{ inputs.db-backend }}" ]]; then + DB_IMAGE="$(docker compose convert --format json | jq -r .services.db.image)" + echo "db-image=$DB_IMAGE" | tee -a "$GITHUB_OUTPUT" + fi + - name: "Load Nautobot App Docker Image" + id: "load" + uses: "./.github/actions/docker-image" + with: + action: "${{ inputs.use-cache == 'true' && 'load-with-cache' || 'load' }}" + image-prefix: "${{ steps.config.outputs.ghcr-image-prefix }}" + image-tag: "${{ steps.config.outputs.ghcr-image-tag }}" + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" + - name: "Tag Nautobot App Docker Image" + shell: "bash" + run: | + cd development + docker tag "${{ steps.config.outputs.ghcr-image }}" "${{ steps.config.outputs.compose-image }}" + - name: "Setup Database Services" + if: | + inputs.db-backend != '' + id: "setup-db" + shell: "bash" + run: | + cd development + docker compose pull -- db redis + docker compose up --detach -- db redis + + if [[ "${{ inputs.use-cache }}" == "true" ]]; then + export NAUTOBOT_VERSION=$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' '${{ steps.config.outputs.ghcr-image }}') + CACHE_KEY="$(docker compose run --rm --entrypoint='' -- nautobot invoke calc-dbdump-cache-key --salt="$NAUTOBOT_VERSION-${{ steps.config.outputs.db-image }}")" + echo "cache-key=db-dump-$CACHE_KEY" | tee -a "$GITHUB_OUTPUT" + fi + - name: "Cache Database Dump" + if: | + inputs.use-cache == 'true' && + inputs.db-backend != '' + id: "cache" + uses: actions/cache@v3 + with: + path: "development/dump.sql" + key: "${{ steps.setup-db.outputs.cache-key }}" + restore-keys: "${{ steps.setup-db.outputs.cache-key }}" + - name: "Use Cached Database Dump" + if: | + inputs.use-cache == 'true' && + inputs.db-backend != '' && + steps.cache.outputs.cache-hit == 'true' + shell: "bash" + run: | + cd development + docker compose exec -- db sh -c \ + '${{ inputs.db-backend == 'mysql' && + 'mysql --user=nautobot --password=$MYSQL_PASSWORD' || + 'psql --username=nautobot postgres' + }}' \ + < dump.sql + - name: "Build Database Dump" + if: | + inputs.use-cache == 'true' && + inputs.db-backend != '' && + steps.cache.outputs.cache-hit != 'true' + shell: "bash" + run: | + cd development + docker compose exec -- db sh -c \ + '${{ inputs.db-backend == 'mysql' && + 'mysql -u nautobot -e "CREATE DATABASE test_nautobot;"' || + 'createdb --user=nautobot test_nautobot' + }}' + docker compose run --rm --entrypoint='' --env=NAUTOBOT_DB_NAME=test_nautobot -- nautobot \ + nautobot-server migrate + docker compose exec -- db sh -c \ + '${{ inputs.db-backend == 'mysql' && + 'mysqldump --user=root --password=$MYSQL_ROOT_PASSWORD --databases test_nautobot' || + 'pg_dump --clean --create --username=nautobot --dbname=test_nautobot' + }}' \ + > dump.sql +outputs: + compose-file: + description: "Docker Compose Files" + value: "${{ steps.config.outputs.compose-file }}" + compose-image: + description: "Docker Compose Image Reference" + value: "${{ steps.config.outputs.compose-image }}" + ghcr-image: + description: "Docker Image Reference" + value: "${{ steps.config.outputs.ghcr-image }}" diff --git a/.github/actions/docker-image/action.yml b/.github/actions/docker-image/action.yml new file mode 100644 index 00000000..4d7d789f --- /dev/null +++ b/.github/actions/docker-image/action.yml @@ -0,0 +1,108 @@ +--- +name: "Build" +description: "Build Nautobot App Docker Image" +inputs: + action: + description: "Action to Perform, (build | load | load-with-cache | pull | push)" + required: true + image-prefix: + description: "Docker Image Prefix" + required: true + image-tag: + description: "Docker Image Tag" + required: true + nautobot-version: + description: "Nautobot Version" + required: true + password: + description: "GitHub Token" + required: false + python-version: + description: "Python Version" + required: true + username: + description: "GitHub Username" + required: false + use-cache: + description: "Use GitHub Actions Cache" + required: false + default: "false" +runs: + using: "composite" + steps: + - name: "Login to GitHub Container Registry" + if: | + inputs.password != '' && + inputs.username != '' + uses: "docker/login-action@v3" + with: + password: "${{ inputs.password }}" + registry: "ghcr.io" + username: "${{ inputs.username }}" + - name: "Set up Docker Buildx" + if: | + inputs.action != 'pull' + uses: "docker/setup-buildx-action@v3" + - name: "Build" + if: | + inputs.action == 'build' + uses: "docker/build-push-action@v5" + with: + context: "./" + file: "./development/Dockerfile" + tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" + cache-from: "type=gha,scope=${{ inputs.image-tag }}" + cache-to: "type=gha,scope=${{ inputs.image-tag }}" + build-args: | + NAUTOBOT_VER=${{ inputs.nautobot-version }} + PYTHON_VER=${{ inputs.python-version }} + - name: "Build and Load" + if: | + inputs.action == 'load' + uses: "docker/build-push-action@v5" + with: + load: true + context: "./" + file: "./development/Dockerfile" + tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" + build-args: | + NAUTOBOT_VER=${{ inputs.nautobot-version }} + PYTHON_VER=${{ inputs.python-version }} + - name: "Build and Load With Cache" + if: | + inputs.action == 'load-with-cache' + uses: "docker/build-push-action@v5" + with: + load: true + context: "./" + file: "./development/Dockerfile" + tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" + cache-from: "type=gha,scope=${{ inputs.image-tag }}" + cache-to: "type=gha,scope=${{ inputs.image-tag }}" + build-args: | + NAUTOBOT_VER=${{ inputs.nautobot-version }} + PYTHON_VER=${{ inputs.python-version }} + - name: "Build and Push" + if: | + inputs.action == 'push' + uses: "docker/build-push-action@v5" + with: + push: true + context: "./" + file: "./development/Dockerfile" + tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" + cache-from: "type=gha,scope=${{ inputs.image-tag }}" + cache-to: "type=gha,scope=${{ inputs.image-tag }}" + build-args: | + NAUTOBOT_VER=${{ inputs.nautobot-version }} + PYTHON_VER=${{ inputs.python-version }} + - name: "Pull" + shell: "bash" + if: | + inputs.action == 'pull' + run: | + docker pull '${{ inputs.image-prefix }}:${{ inputs.image-tag }}' +outputs: + image: + description: "Docker Image Reference" + value: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" diff --git a/.github/actions/run-linters/action.yml b/.github/actions/run-linters/action.yml new file mode 100644 index 00000000..cad62175 --- /dev/null +++ b/.github/actions/run-linters/action.yml @@ -0,0 +1,31 @@ +--- +name: "Run Linters" +description: "Run Linters for Nautobot App" +inputs: + compose-file: + description: "Docker Compose File" + required: true + nautobot-version: + description: "Nautobot Version" + required: true + python-version: + description: "Python Version" + required: true +runs: + using: "composite" + steps: + - name: "Run Linters" + shell: "bash" + env: + COMPOSE_ANSI: "0" + COMPOSE_FILE: "${{ inputs.compose-file }}" + NAUTOBOT_VER: "${{ inputs.nautobot-version }}" + PYTHON_VER: "${{ inputs.python-version }}" + run: | + cd development + docker-compose run \ + --rm \ + --entrypoint='' \ + -- \ + nautobot \ + invoke tests --lint-only --no-test-docs diff --git a/.github/actions/unittests/action.yml b/.github/actions/unittests/action.yml new file mode 100644 index 00000000..33d41fb5 --- /dev/null +++ b/.github/actions/unittests/action.yml @@ -0,0 +1,31 @@ +--- +name: "Unit Tests" +description: "Run Unit Tests for Nautobot App" +inputs: + compose-file: + description: "Docker Compose Files" + required: true + nautobot-version: + description: "Nautobot Version" + required: true + python-version: + description: "Python Version" + required: true +runs: + using: "composite" + steps: + - name: "Unit Tests" + shell: "bash" + env: + COMPOSE_ANSI: "0" + COMPOSE_FILE: "${{ inputs.compose-file }}" + NAUTOBOT_VER: "${{ inputs.nautobot-version }}" + PYTHON_VER: "${{ inputs.python-version }}" + run: | + cd development + docker compose run \ + --rm \ + --entrypoint='' \ + -- \ + nautobot \ + invoke unittest --failfast --keepdb diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 00000000..984e7718 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,42 @@ +# NOT USED +--- +name: "Build Image" +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + nautobot-version: + description: "Nautobot Version" + required: true + type: "string" + python-version: + description: "Python Version" + required: true + type: "string" + tag-prefix: + description: "Docker Image Tag Prefix" + required: true + type: "string" + # username: + # description: "GitHub Username" + # required: true + # type: "string" + # secrets: + # password: + # description: "GitHub Token" + # required: true +jobs: + build: + runs-on: "ubuntu-22.04" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Build Nautobot App Docker Image" + uses: "./.github/actions/docker-image" + with: + action: "build" + image-prefix: "ghcr.io/${{ github.repository }}/nautobot-dev-test" + image-tag: "${{ inputs.tag-prefix }}-${{ inputs.nautobot-version }}-py${{ inputs.python-version }}" + nautobot-version: "${{ inputs.nautobot-version }}" + # password: "${{ secrets.password }}" + python-version: "${{ inputs.python-version }}" + # username: "${{ inputs.username }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b539e935..baea6643 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,14 +3,17 @@ name: "CI" concurrency: # Cancel any existing runs of this workflow for this same PR group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: true -on: # yamllint disable-line rule:truthy rule:comments - push: - branches: - - "main" - - "develop" - tags: - - "v*" - pull_request: ~ +# yamllint disable-line rule:truthy rule:comments +on: "workflow_dispatch" +# TODO: Temporary disabled +# on: +# push: +# branches: +# - "main" +# - "develop" +# tags: +# - "v*" +# pull_request: ~ env: PLUGIN_NAME: "nautobot-plugin-firewall-models" diff --git a/.github/workflows/full-tests-parallel.yml b/.github/workflows/full-tests-parallel.yml new file mode 100644 index 00000000..15a94f94 --- /dev/null +++ b/.github/workflows/full-tests-parallel.yml @@ -0,0 +1,61 @@ +--- +name: "Full Tests Parallel" +on: # yamllint disable-line rule:truthy + # pull_request: {} + workflow_dispatch: {} +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true +jobs: + # build: + # strategy: + # fail-fast: true + # matrix: + # nautobot-version: ["stable"] + # # python-version: ["3.11"] + # python-version: ["3.8", "3.11"] + # include: + # - nautobot-version: "2.0.0" + # python-version: "3.11" + # uses: "./.github/workflows/build-image.yml" + # with: + # nautobot-version: "${{ matrix.nautobot-version }}" + # python-version: "${{ matrix.python-version }}" + # tag-prefix: "pr-test-${{ github.event.pull_request.number }}" + # # username: "${{ github.actor }}" + # # secrets: + # # password: "${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}" + linters: + # needs: "build" + strategy: + fail-fast: true + matrix: + nautobot-version: ["stable"] + python-version: ["3.11"] + uses: "./.github/workflows/run-linters.yml" + with: + nautobot-version: "${{ matrix.nautobot-version }}" + python-version: "${{ matrix.python-version }}" + tag-prefix: "pr-${{ github.event.pull_request.number }}" + unittest: + # needs: "build" + strategy: + fail-fast: true + matrix: + db-backend: ["postgres"] + nautobot-version: ["stable"] + # python-version: ["3.11"] + python-version: ["3.8", "3.11"] + include: + - db-backend: "postgres" + nautobot-version: "2.0.0" + python-version: "3.11" + # - db-backend: "mysql" + # nautobot-version: "2.0.0" + # python-version: "3.11" + uses: "./.github/workflows/unittest.yml" + with: + db-backend: "${{ matrix.db-backend }}" + nautobot-version: "${{ matrix.nautobot-version }}" + python-version: "${{ matrix.python-version }}" + tag-prefix: "pr-${{ github.event.pull_request.number }}" diff --git a/.github/workflows/full-tests-simple.yml b/.github/workflows/full-tests-simple.yml new file mode 100644 index 00000000..7a78a335 --- /dev/null +++ b/.github/workflows/full-tests-simple.yml @@ -0,0 +1,99 @@ +--- +name: "Full Tests Simple" + +on: # yamllint disable-line rule:truthy + # pull_request: {} + workflow_dispatch: {} + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +env: + DISABLE_MYSQL_TESTS: 1 + +jobs: + tests: + runs-on: "ubuntu-22.04" + strategy: + fail-fast: true + matrix: + nautobot-version: ["stable"] + python-version: ["3.11", "3.8"] + include: + - nautobot-version: "2.0.0" + python-version: "3.11" + env: + NAUTOBOT_VER: "${{ matrix.nautobot-version }}" + PYTHON_VER: "${{ matrix.python-version }}" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Configure" + id: "config" + env: + COMPOSE_FILE: "docker-compose.base.yml" + run: | + cd development + cp creds.example.env creds.env + # Read Docker image reference from Compose configuration + IMAGE="$(docker compose convert --format json | jq -r .services.nautobot.image)" + cd - + + RUN_LINTERS=0 + TEST_MYSQL=0 + if [[ "${{ matrix.python-version }}" == "3.11" ]]; then + if [[ "${{ matrix.nautobot-version }}" == "stable" ]]; then + RUN_LINTERS=1 + else + if [[ "${{ env.DISABLE_MYSQL_TESTS }}" != "1" ]]; then + TEST_MYSQL=1 + fi + fi + fi + + echo "image=$IMAGE" | tee -a "$GITHUB_OUTPUT" + echo "run-linters=$RUN_LINTERS" | tee -a "$GITHUB_OUTPUT" + echo "test-mysql=$TEST_MYSQL" | tee -a "$GITHUB_OUTPUT" + - name: "Build" + uses: "./.github/actions/build" + with: + image: "${{ steps.config.outputs.image }}" + nautobot-version: "${{ matrix.nautobot-version }}" + python-version: "${{ matrix.python-version }}" + - name: "Run Linters" + if: | + steps.config.outputs.run-linters == '1' + run: | + docker run \ + --rm \ + --entrypoint='' \ + --volume="$PWD:/source" \ + --env-file=development/creds.env \ + --env-file=development/development.env \ + '${{ steps.config.outputs.image }}' \ + invoke tests --lint-only --no-test-docs + - name: "Test with Postgres" + env: + COMPOSE_FILE: "docker-compose.base.yml:docker-compose.postgres.yml:docker-compose.redis.yml:docker-compose.dev.yml" + run: | + cd development + docker compose run \ + --rm \ + --entrypoint='' \ + -- \ + nautobot \ + invoke unittest --failfast + - name: "Test with MySQL" + if: | + steps.config.outputs.test-mysql == '1' + env: + COMPOSE_FILE: "docker-compose.base.yml:docker-compose.mysql.yml:docker-compose.redis.yml:docker-compose.dev.yml" + run: | + cd development + docker compose run \ + --rm \ + --entrypoint='' \ + -- \ + nautobot \ + invoke unittest --failfast diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 00000000..c9138ba2 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,13 @@ +--- +name: "Test Pull Request" +# yamllint disable-line rule:truthy +on: "pull_request" +jobs: + tests: + uses: "./.github/workflows/single-test-parallel.yml" + with: + db-backend: "postgres" + nautobot-version: "stable" + python-version: "3.11" + tag-prefix: "pr-${{ github.event.pull_request.number }}" + use-cache: true diff --git a/.github/workflows/rebake.yml b/.github/workflows/rebake.yml index b4688842..13d1e3a0 100644 --- a/.github/workflows/rebake.yml +++ b/.github/workflows/rebake.yml @@ -30,7 +30,7 @@ on: # yamllint disable-line rule:truthy drift-manager-tag: description: "The drift manager Docker image tag to use" type: "string" - default: "prod" + default: "latest" workflow_dispatch: inputs: cookie: @@ -60,7 +60,7 @@ on: # yamllint disable-line rule:truthy drift-manager-tag: description: "The drift manager Docker image tag to use" type: "string" - default: "prod" + default: "latest" jobs: rebake: runs-on: "ubuntu-22.04" diff --git a/.github/workflows/run-linters.yml b/.github/workflows/run-linters.yml new file mode 100644 index 00000000..c3594f26 --- /dev/null +++ b/.github/workflows/run-linters.yml @@ -0,0 +1,42 @@ +--- +name: "Run Linters" +# yamllint disable-line rule:truthy +on: + workflow_call: + inputs: + nautobot-version: + description: "Nautobot Version" + required: true + type: "string" + python-version: + description: "Python Version" + required: true + type: "string" + tag-prefix: + description: "Docker Image Tag Prefix" + required: true + type: "string" + use-cache: + description: "Whether to use GitHub Actions cache" + required: true + type: "boolean" +jobs: + linters: + runs-on: "ubuntu-22.04" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Configure Compose" + id: "config" + uses: "./.github/actions/config-compose" + with: + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" + tag-prefix: "${{ inputs.tag-prefix }}" + use-cache: "${{ inputs.use-cache }}" + - name: "Run Linters" + uses: "./.github/actions/run-linters" + with: + compose-file: "${{ steps.config.outputs.compose-file }}" + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" diff --git a/.github/workflows/single-test-parallel.yml b/.github/workflows/single-test-parallel.yml new file mode 100644 index 00000000..cb6f1db8 --- /dev/null +++ b/.github/workflows/single-test-parallel.yml @@ -0,0 +1,42 @@ +--- +name: "Test Single Nautobot and Python Version in Parallel" +# yamllint disable-line rule:truthy +on: + workflow_call: + inputs: + db-backend: + description: "The database backend to test against" + required: true + type: "string" + nautobot-version: + description: "The Nautobot version to test against" + required: true + type: "string" + python-version: + description: "The Python version to test against" + required: true + type: "string" + tag-prefix: + description: "The prefix to use for the tag" + required: true + type: "string" + use-cache: + description: "Whether to use GitHub Actions cache" + required: true + type: "boolean" +jobs: + linters: + uses: "./.github/workflows/run-linters.yml" + with: + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" + tag-prefix: "${{ inputs.tag-prefix }}" + use-cache: "${{ inputs.use-cache }}" + unittest: + uses: "./.github/workflows/unittest.yml" + with: + db-backend: "${{ inputs.db-backend }}" + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" + tag-prefix: "${{ inputs.tag-prefix }}" + use-cache: "${{ inputs.use-cache }}" diff --git a/.github/workflows/single-test-simple.yml b/.github/workflows/single-test-simple.yml new file mode 100644 index 00000000..e58b7b09 --- /dev/null +++ b/.github/workflows/single-test-simple.yml @@ -0,0 +1,53 @@ +--- +name: "Single Test Simple" + +on: # yamllint disable-line rule:truthy + # pull_request: {} + workflow_dispatch: {} + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +env: + NAUTOBOT_VER: "stable" + PYTHON_VER: "3.11" + RUN_MYSQL_TESTS: "false" + +jobs: + tests: + runs-on: "ubuntu-22.04" + steps: + - name: "Check Out Repository Code" + uses: "actions/checkout@v4" + - name: "Build Nautobot App Docker Image" + uses: "./.github/actions/docker-image" + with: + action: "load" + image-prefix: "ghcr.io/${{ github.repository }}/nautobot-dev-test" + image-tag: "pr-${{ github.event.pull_request.number }}-${{ env.NAUTOBOT_VER }}-py${{ env.PYTHON_VER }}" + nautobot-version: "${{ env.NAUTOBOT_VER }}" + # password: "${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}" + # pull: true + # push: true + python-version: "${{ env.PYTHON_VER }}" + # username: "${{ github.actor }}" + - name: "Run Linters" + uses: "./.github/actions/run-linters" + with: + nautobot-version: "${{ env.NAUTOBOT_VER }}" + python-version: "${{ env.PYTHON_VER }}" + - name: "Unit Tests with MySQL" + if: | + env.RUN_MYSQL_TESTS == 'true' + uses: "./.github/actions/unittests" + with: + db-backend: "mysql" + nautobot-version: "${{ env.NAUTOBOT_VER }}" + python-version: "${{ env.PYTHON_VER }}" + - name: "Unit Tests with PostgreSQL" + uses: "./.github/actions/unittests" + with: + db-backend: "postgres" + nautobot-version: "${{ env.NAUTOBOT_VER }}" + python-version: "${{ env.PYTHON_VER }}" diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml new file mode 100644 index 00000000..ae92c03b --- /dev/null +++ b/.github/workflows/unittest.yml @@ -0,0 +1,47 @@ +--- +name: "Unit Tests" +# yamllint disable-line rule:truthy +on: + workflow_call: + inputs: + db-backend: + description: "Database Backend" + required: true + type: "string" + nautobot-version: + description: "Nautobot Version" + required: true + type: "string" + python-version: + description: "Python Version" + required: true + type: "string" + tag-prefix: + description: "Docker Image Tag Prefix" + required: true + type: "string" + use-cache: + description: "Whether to use GitHub Actions cache" + required: true + type: "boolean" +jobs: + unittest: + runs-on: "ubuntu-22.04" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Configure Compose" + id: "config" + uses: "./.github/actions/config-compose" + with: + db-backend: "${{ inputs.db-backend }}" + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" + tag-prefix: "${{ inputs.tag-prefix }}" + use-cache: "${{ inputs.use-cache }}" + - name: "Unit Tests" + uses: "./.github/actions/unittests" + with: + compose-file: "${{ steps.config.outputs.compose-file }}" + nautobot-version: "${{ inputs.nautobot-version }}" + python-version: "${{ inputs.python-version }}" diff --git a/ci.md b/ci.md new file mode 100644 index 00000000..c2fb75c3 --- /dev/null +++ b/ci.md @@ -0,0 +1,229 @@ +# Nautobot App CI Proposal + +A proposal to improve GitHub based CI for Nautobot App development. + +## Processes + +Description of processes to be implemented. Bullets under `Trigger GitHub workflow ...` are automated GitHub workflow jobs and steps. + +### Add Feature + +Process of developing a new feature. + +The idea is to run as simple as possible tests for each pull request commit, and all tests only once for merge commit. + +- Locally run `invoke add-feature --issue <#issue>` to start developing a new feature. This task will: + - Fetch the repository remote branches and tags. + - Checkout and pull the latest `develop`. + - Create a new feature branch `u/-<#issue>-`. + - Omit `#issue` if not related to any issue. + - Add `towncrier` fragment. + - Commit and push the feature branch. + - Open a new pull request to `develop` branch. +- Implement the feature (assignee). +- Trigger GitHub workflow by pushing a commit to the feature branch. + - Build the documentation using readthedocs.org. + - Test `towncrier` fragment existence. + - Run linters. + - Run unit tests using latest `stable` Nautobot version, latest supported Python version and PostgreSQL. +- Review and approve the pull request (code owner). +- Squash and merge the pull requests (assignee). +- Trigger GitHub workflow by merging to `develop`. + - [Fully test](#full-tests) the commit. + - If full tests fail, trigger GitHub workflow to [fix failed merge](#fix-failed-merge). + +#### How is this different from the current/existing CI workflow? + +Here is an examples run, still WIP: + +https://github.com/nautobot/nautobot-plugin-firewall-models/actions/runs/6470190422 + +- All linters run as a single job. +- Linters use the same docker image as unit tests, no need to install dependencies using poetry. +- Unit tests run only one test using latest `stable` Nautobot version, latest supported Python version and PostgreSQL. + +Speedup against current solution is about 40 % and uses significantly fewer workers. + +Docker caching is [explained here](#docker-caching). + +### Bug Fix + +The process is the same as [adding a new feature](#add-feature). + +### Stable Release + +To safely release a new stable version. + +- Locally run `invoke release --version 'X.Y.Z' --ref <git reference> --push` to start the release process. This task will: + - Fetch the repository remote branches and tags. + - Pull the latest `develop` and `main`. + - Fail if tag `vX.Y.Z` already exists. + - Update `pyproject.toml` version to the version provided. + - Implement some checking between the current vs provided versions. + - Checkout and rebase `main` to the `--ref` argument value. + - Default value is the latest `develop`. + - Fail if provided git reference is not a descendant of `main` or `develop`. + - Create changelog based on `towncrier` fragments. + - Commit and push the `main` branch. + - Open a new pull request to `develop` branch. +- Trigger GitHub workflow by pushing a commit to the release pull request. + - [Fully test](#full-tests) the commit. +- Review and approve the pull request (code owner). +- Trigger GitHub workflow by approving the release pull request: + - Check whether [full tests](#full-tests) for the commit passed. + - Tag the commit `vX.Y.Z` and push the tag. +- Trigger GitHub workflow by pushing a tag: + - Check whether [full tests](#full-tests) for the tagged commit passed. + - The commit should be tagged in the previous step only if the tests passed, however, it is possible to tag the commit manually. This will verify it. + - Build a package. + - Create a new GitHub release. +- Trigger GitHub workflow by creating a GitHub release. + - Release the package to PyPI. + - Merge and close the release pull request. Do not squash to keep the release commit history. + +When some step fails, it can be simply re-run. + +### Pre Release + +To be able to quickly release a new pre-release version. + +Similar to [stable release](#stable-release), but: + +- Locally run `invoke pre-release --version <version> --base <branch name> --push` to start the release process. This task will: + - Increment the version if no `--version` argument is provided, e.g.: + - `1.0.1` => `1.0.2-dev0` + - `1.0.2-dev0` => `1.0.2-dev1` + - An example implementation is [here](https://github.com/nautobot/cookiecutter-nautobot-app-drift-manager/blob/develop/tasks.py#L166). + - Create a new `u/<username>-v<version>` branch from the current git reference. + - Open a new pull request to the `--base` branch. + - Use the current branch as the base branch if no `--base` argument is provided. +- Approval can be done by pull request author, if the base branch is not protected. +- Delete the release branch after successful release. + +### Bug Fix LTM + +Implement and merge bug fix to `develop` first, if the bug is present in both, stable and LTM releases. + +- Locally run `invoke fix-ltm --ref <merge-commit-sha | #issue>` to start fixing LTM bug. This task will: + - Fetch the repository remote branches and tags. + - Checkout and pull the latest `ltm-1.6`. + - Create a new branch `v/<user name>-<#issue>-<title>. + - Increment the patch version in `pyproject.toml`. + - Cherry-pick the commit from `develop` if provided by `--ref`. + - Merge commit reference can be determined from #issue and vice versa. + - Commit and push the branch. + - Open a new pull request to `ltm-1.6` branch. +- If the bug is not present in stable release, implement and commit the bug fix (assignee). +- Trigger GitHub workflow by pushing a commit to the feature branch. + - [Fully test](#full-tests) the commit. +- Review and approve the pull request (code owner). +- Squash and merge the pull requests (assignee). + +### Back-port Feature to LTM + +If allowed, process will be the same as [bug fix LTM](#bug-fix-ltm). + +### LTM Release + +To safely release a new LTM version. + +Similar to [stable release](#stable-release), but: + +- The invoke task is named `invoke release-ltm`. + - `--version` argument is missing. + - Increment version `patch` part only. +- Use `ltm-1.6` branch instead of `develop`. +- Use protected `ltm-1.6-main` branch (doesn't exist yet) instead of `main`. + +Considerations: + +- Align branch names: + - Rename `ltm-1.6` => `ltm-1.6/develop`. + - Rename `ltm-1.6-main` => `ltm-1.6/main`. + - Use `ltm-1.6/u/...` for feature branches. +- Process can be automated for each [LTM bug fix](#bug-fix-ltm) to speed things up. + +### Fix Failed Merge + +It's rare but possible, that after the merge to the latest `develop`, something can get broken, even when tests on feature branch passes. E.g.: incompatibility between concurrent features. + +When the full tests fail, the following steps will be done automatically by GitHub workflow: + +- Create a new pull request with rollback commit. +- Re-open the feature pull request. + - An option is to open a new issue instead. + +It's up to the users to decide, whether to fix the failed merge or not. + +## GitHub Actions + +Define reusable actions to be used by workflows. + +Actions `.yml` files can be stored in the following locations: + +- `.github/actions` folder for each repository and managed by the Drift Manager. +- Some public shared repository (e.g. `cookiecutter-nautobot-app/` can be used after open-sourcing). + +The following actions can be defined: + +- Build Docker image for specific Python and Nautobot version. +- Run linters for specific Python and Nautobot version. +- Run unit tests for specific database type, Python and Nautobot version. +- Full tests as [described here](#full-tests). +- Build a package. +- Release a tag to GitHub. +- Release a package to PyPI. + +### Full Tests + +Action, that contains the following tests: + +- Build the documentation using readthedocs.org. +- Python 3.11, Nautobot latest `stable` linters. +- Python 3.11, Nautobot latest `stable`, PostgreSQL unit tests. +- Python 3.11, Nautobot latest `stable`, MySQL unit tests. +- Python 3.11, Nautobot `2.0.0`, PostgreSQL unit tests. +- Python 3.8, Nautobot latest `stable`, PostgreSQL unit tests. + +Jobs will run in parallel and re-use cached Docker layers and [database dumps](#database-caching). + +## Docker Caching + +When testing the single Nautobot and Python version the cache is much less utilized. + +- Currently, the limit is 10 G per repository. Each Nautobot/Python combination uses about 2.5 GB. That means, two concurrent PRs will purge others cache. +- For full tests, caching will be disabled. + +Better define `.gitignore` file to avoid unnecessary context changes. + +- Deny everything first. +- Allow particular files/directories necessary for build. + +## Database Caching + +Cache and re-use empty migrated database dumps to avoid migrations using GitHub actions cache. + +GitHub `unittest` action will first check, whether there is cached dump. + +- If so, apply that dump. +- If not, run migrations, create a new dump, and cache that dump. + +Unit tests will be run with `--keepdb` flag to avoid re-creating the database. + +Calculate cache key as a hash of: + +- `migrations` folder file content. +- Nautobot version. +- Database server Docker image reference. + +This should speed up unit tests significantly. + +## Future Improvements + +- Add E2E Selenium tests. +- Add E2E external integrations tests. +- Factory dumps caching similar to Nautobot core. + +## Questions + +- [ ] What is preferred in GitHub workflows, to fail fast or finish fast? diff --git a/development/Dockerfile b/development/Dockerfile index 37c07122..54218d9c 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -50,8 +50,11 @@ RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ # !!! USE CAUTION WHEN MODIFYING LINES BELOW # Copy in the source code +COPY pyproject.toml poetry.lock /source/ WORKDIR /source -COPY . /source +RUN touch README.md && \ + mkdir nautobot_firewall_models && \ + touch nautobot_firewall_models/__init__.py # Get container's installed Nautobot version as a forced constraint # NAUTOBOT_VER may be a branch name and not a published release therefor we need to get the installed version @@ -76,6 +79,4 @@ RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ # Can be improved in Poetry 1.2 which allows `poetry install --only dev` RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ pip install -c constraints.txt -r poetry_freeze_dev.txt - -COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py # !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/Dockerfile.dockerignore b/development/Dockerfile.dockerignore new file mode 100644 index 00000000..fffef8cb --- /dev/null +++ b/development/Dockerfile.dockerignore @@ -0,0 +1,4 @@ +* + +!/pyproject.toml +!/poetry.lock diff --git a/development/docker-compose.base.yml b/development/docker-compose.base.yml index af94538c..2d27eb68 100644 --- a/development/docker-compose.base.yml +++ b/development/docker-compose.base.yml @@ -16,11 +16,6 @@ x-nautobot-base: &nautobot-base version: "3.8" services: nautobot: - depends_on: - redis: - condition: "service_started" - db: - condition: "service_healthy" <<: - *nautobot-base - *nautobot-build diff --git a/development/docker-compose.mysql.yml b/development/docker-compose.mysql.yml index 062ada94..94b9c59c 100644 --- a/development/docker-compose.mysql.yml +++ b/development/docker-compose.mysql.yml @@ -9,6 +9,11 @@ services: - "development.env" - "creds.env" - "development_mysql.env" + depends_on: + redis: + condition: "service_started" + db: + condition: "service_healthy" worker: environment: - "NAUTOBOT_DB_ENGINE=django.db.backends.mysql" diff --git a/development/docker-compose.postgres.yml b/development/docker-compose.postgres.yml index 12d1de31..7a086397 100644 --- a/development/docker-compose.postgres.yml +++ b/development/docker-compose.postgres.yml @@ -5,6 +5,11 @@ services: nautobot: environment: - "NAUTOBOT_DB_ENGINE=django.db.backends.postgresql" + depends_on: + redis: + condition: "service_started" + db: + condition: "service_healthy" db: image: "postgres:13-alpine" command: diff --git a/tasks.py b/tasks.py index 17cda20c..573374b1 100644 --- a/tasks.py +++ b/tasks.py @@ -13,6 +13,8 @@ """ import os +from hashlib import sha256 +from pathlib import Path from invoke.collection import Collection from invoke.tasks import task as invoke_task @@ -666,9 +668,10 @@ def unittest_coverage(context): "failfast": "fail as soon as a single test fails don't run the entire test suite. (default: False)", "keepdb": "Save and re-use test database between test runs for faster re-testing. (default: False)", "lint-only": "Only run linters; unit tests will be excluded. (default: False)", + "test-docs": "Build documentation to be available within Nautobot. (default: True)", } ) -def tests(context, failfast=False, keepdb=False, lint_only=False): +def tests(context, failfast=False, keepdb=False, lint_only=False, test_docs=True): """Run all tests for this plugin.""" # If we are not running locally, start the docker containers so we don't have to for each test if not is_truthy(context.nautobot_firewall_models.local): @@ -691,10 +694,34 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): check_migrations(context) print("Running pylint...") pylint(context) - print("Running mkdocs...") - build_and_check_docs(context) + if test_docs: + print("Running mkdocs...") + build_and_check_docs(context) if not lint_only: print("Running unit tests...") unittest(context, failfast=failfast, keepdb=keepdb) unittest_coverage(context) print("All tests have passed!") + + +@task(help={"salt": "Salt to use when generating cache key."}) +def calc_dbdump_cache_key(_context, salt=""): + """Calculate database dump cache key. + + Calculate cache key as: + + - `migrations` folder file content. + - `salt` argument. + """ + migrations_dir = Path("nautobot_firewall_modesl/migrations") + hasher = sha256() + + hasher.update(salt.encode()) + + if migrations_dir.is_dir(): + for file in sorted(migrations_dir.rglob("*.py")): + if file.is_file(): + with file.open("r") as f: + hasher.update(f.read().encode()) + + print(hasher.hexdigest())