From fc0133e886d3dbf7364ddd24915844dda0650e4f Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 18 Oct 2023 07:16:02 +0100 Subject: [PATCH 1/2] ref(workflows): consolidate workflows based on their purpose (#7616) * ref(workflows): consolidate workflows based on their purpose This also renames the workflows to make their naming more consistent and adding a naming convention Fixes: #6166 Fixes: #6167 * fix(workflows): use correct name for patch * fix(workflow): docker unit tests * fix(release): validation error Error: ``` Validation Failed: {"resource":"Release","code":"invalid","field":"target_commitish"} ``` Fixes: https://github.com/release-drafter/release-drafter/issues/1125 * fix(workflows): reference correct name * fix: remove extra workflow * fix(workflows): use larger runners * fix(workflow): remove code already in docker unit-test * fix(unit-tests): start zebra the right way * fix: typo in patch name * chore: move job to logical order * imp(workflows): use better name for gcp tests * add: missing merge changes * chore: use better name for find-disks * fix(ci): use the `entrypoint.sh` to change the Network * fix(ci): add missing `ZEBRA_CONF_PATH` variable * fix(ci): allow to build the entrypoint file with testnet * fix(entrypoint): allow to create a dir and file with a single variable * refactor: test config file in CI and CD with a reusable workflow * fix(ci): wrong name used * fix(ci): use checkout * fix(ci): improve docker config tests * fix(ci): use better name for protection rules * Fix changed workflow file name in docs patch file * Apply suggestions from code review Co-authored-by: teor Co-authored-by: Arya * fix(cd): depend on file tests * fix(docs): adapt to new workflow name * fix: revert test coverage on CD * chore: reduce diff * fix(ci): allow using variable images for reusable workflows * fix(dockerfile): use variables or default for config path and file * fix(entrypoint): if `$ZEBRA_CONF_PATH` is set, do not override it * Fix patch job names and remove failure job testnet dependencies --------- Co-authored-by: teor Co-authored-by: Arya --- .../workflows/cd-deploy-nodes-gcp.patch.yml | 39 +++ ...s-delivery.yml => cd-deploy-nodes-gcp.yml} | 138 +++------ ...ces.yml => chore-delete-gcp-resources.yml} | 0 ...ement.yml => chore-project-management.yml} | 0 ...ly.patch.yml => ci-build-crates.patch.yml} | 2 +- ...s-individually.yml => ci-build-crates.yml} | 4 +- ...verage.patch.yml => ci-coverage.patch.yml} | 2 +- .../{coverage.yml => ci-coverage.yml} | 4 +- ...yml => ci-integration-tests-gcp.patch.yml} | 52 +--- ...ocker.yml => ci-integration-tests-gcp.yml} | 253 ++-------------- .../{lint.patch.yml => ci-lint.patch.yml} | 0 .github/workflows/{lint.yml => ci-lint.yml} | 2 +- .../workflows/ci-unit-tests-docker.patch.yml | 82 ++++++ .github/workflows/ci-unit-tests-docker.yml | 269 ++++++++++++++++++ ...s.patch.yml => ci-unit-tests-os.patch.yml} | 4 +- ...ntegration-os.yml => ci-unit-tests-os.yml} | 31 +- .../workflows/continous-delivery.patch.yml | 35 --- ...tinous-integration-docker.patch-always.yml | 26 -- ...tch.yml => docs-deploy-firebase.patch.yml} | 2 +- .../{docs.yml => docs-deploy-firebase.yml} | 2 +- ...ion.yml => docs-dockerhub-description.yml} | 0 ...al-deploy.yml => manual-zcashd-deploy.yml} | 0 .github/workflows/release-binaries.yml | 6 +- .github/workflows/release-drafter.yml | 1 + ...r-image.yml => sub-build-docker-image.yml} | 0 ...h.yml => sub-build-lightwalletd.patch.yml} | 4 +- ...walletd.yml => sub-build-lightwalletd.yml} | 6 +- ...-params.yml => sub-build-zcash-params.yml} | 8 +- ...l => sub-deploy-integration-tests-gcp.yml} | 8 +- ...ed-disks.yml => sub-find-cached-disks.yml} | 0 .github/workflows/sub-test-zebra-config.yml | 79 +++++ book/src/dev/continuous-integration.md | 2 +- docker/Dockerfile | 6 +- docker/entrypoint.sh | 5 +- 34 files changed, 595 insertions(+), 477 deletions(-) create mode 100644 .github/workflows/cd-deploy-nodes-gcp.patch.yml rename .github/workflows/{continous-delivery.yml => cd-deploy-nodes-gcp.yml} (77%) rename .github/workflows/{delete-gcp-resources.yml => chore-delete-gcp-resources.yml} (100%) rename .github/workflows/{project-management.yml => chore-project-management.yml} (100%) rename .github/workflows/{build-crates-individually.patch.yml => ci-build-crates.patch.yml} (97%) rename .github/workflows/{build-crates-individually.yml => ci-build-crates.yml} (98%) rename .github/workflows/{coverage.patch.yml => ci-coverage.patch.yml} (93%) rename .github/workflows/{coverage.yml => ci-coverage.yml} (97%) rename .github/workflows/{continous-integration-docker.patch.yml => ci-integration-tests-gcp.patch.yml} (67%) rename .github/workflows/{continous-integration-docker.yml => ci-integration-tests-gcp.yml} (71%) rename .github/workflows/{lint.patch.yml => ci-lint.patch.yml} (100%) rename .github/workflows/{lint.yml => ci-lint.yml} (99%) create mode 100644 .github/workflows/ci-unit-tests-docker.patch.yml create mode 100644 .github/workflows/ci-unit-tests-docker.yml rename .github/workflows/{continous-integration-os.patch.yml => ci-unit-tests-os.patch.yml} (96%) rename .github/workflows/{continous-integration-os.yml => ci-unit-tests-os.yml} (97%) delete mode 100644 .github/workflows/continous-delivery.patch.yml delete mode 100644 .github/workflows/continous-integration-docker.patch-always.yml rename .github/workflows/{docs.patch.yml => docs-deploy-firebase.patch.yml} (94%) rename .github/workflows/{docs.yml => docs-deploy-firebase.yml} (99%) rename .github/workflows/{dockerhub-description.yml => docs-dockerhub-description.yml} (100%) rename .github/workflows/{zcashd-manual-deploy.yml => manual-zcashd-deploy.yml} (100%) rename .github/workflows/{build-docker-image.yml => sub-build-docker-image.yml} (100%) rename .github/workflows/{zcash-lightwalletd.patch.yml => sub-build-lightwalletd.patch.yml} (85%) rename .github/workflows/{zcash-lightwalletd.yml => sub-build-lightwalletd.yml} (97%) rename .github/workflows/{zcash-params.yml => sub-build-zcash-params.yml} (86%) rename .github/workflows/{deploy-gcp-tests.yml => sub-deploy-integration-tests-gcp.yml} (99%) rename .github/workflows/{find-cached-disks.yml => sub-find-cached-disks.yml} (100%) create mode 100644 .github/workflows/sub-test-zebra-config.yml diff --git a/.github/workflows/cd-deploy-nodes-gcp.patch.yml b/.github/workflows/cd-deploy-nodes-gcp.patch.yml new file mode 100644 index 00000000000..457bedc2a3b --- /dev/null +++ b/.github/workflows/cd-deploy-nodes-gcp.patch.yml @@ -0,0 +1,39 @@ +name: Deploy Nodes to GCP + +on: + pull_request: + paths-ignore: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/cd-deploy-nodes-gcp.yml' + - '.github/workflows/sub-build-docker-image.yml' + +jobs: + build: + name: Build CD Docker / Build images + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-configuration-file: + name: Test CD default Docker config file / Test default-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-zebra-conf-path: + name: Test CD custom Docker config file / Test custom-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' \ No newline at end of file diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/cd-deploy-nodes-gcp.yml similarity index 77% rename from .github/workflows/continous-delivery.yml rename to .github/workflows/cd-deploy-nodes-gcp.yml index 0b9a888c6ed..0b167ceb63c 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/cd-deploy-nodes-gcp.yml @@ -1,4 +1,4 @@ -name: CD +name: Deploy Nodes to GCP # Ensures that only one workflow task will run at a time. Previous deployments, if # already in process, won't get cancelled. Instead, we let the first to complete @@ -47,8 +47,8 @@ on: # # workflow definitions # - 'docker/**' # - '.dockerignore' - # - '.github/workflows/continous-delivery.yml' - # - '.github/workflows/build-docker-image.yml' + # - '.github/workflows/cd-deploy-nodes-gcp.yml' + # - '.github/workflows/sub-build-docker-image.yml' # Only runs the Docker image tests, doesn't deploy any instances pull_request: @@ -66,14 +66,13 @@ on: # workflow definitions - 'docker/**' - '.dockerignore' - - '.github/workflows/continous-delivery.yml' - - '.github/workflows/find-cached-disks.yml' + - '.github/workflows/cd-deploy-nodes-gcp.yml' + - '.github/workflows/sub-build-docker-image.yml' release: types: - published - jobs: # If a release was made we want to extract the first part of the semver from the # tag_name @@ -108,7 +107,7 @@ jobs: # The image will be commonly named `zebrad:` build: name: Build CD Docker - uses: ./.github/workflows/build-docker-image.yml + uses: ./.github/workflows/sub-build-docker-image.yml with: dockerfile_path: ./docker/Dockerfile dockerfile_target: runtime @@ -118,99 +117,40 @@ jobs: # Test that Zebra works using the default config with the latest Zebra version. test-configuration-file: - name: Test Zebra CD Docker config file - timeout-minutes: 15 - runs-on: ubuntu-latest + name: Test CD default Docker config file needs: build - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - # Make sure Zebra can sync at least one full checkpoint on mainnet - - name: Run tests using the default config - shell: /usr/bin/bash -exo pipefail {0} - run: | - docker pull ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - docker run --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - - # Use a subshell to handle the broken pipe error gracefully - ( - trap "" PIPE; - docker logs \ - --tail all \ - --follow \ - default-conf-tests | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter" - ) || true - LOGS_EXIT_STATUS=$? - - docker stop default-conf-tests - - EXIT_STATUS=$(docker wait default-conf-tests || echo "Error retrieving exit status"); - echo "docker exit status: $EXIT_STATUS"; - - # If grep found the pattern, exit with the Docker container exit status - if [ $LOGS_EXIT_STATUS -eq 0 ]; then - exit $EXIT_STATUS; - fi - - # Handle other potential errors here - echo "An error occurred while processing the logs."; - exit 1; + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: 'default-conf' + docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} + grep_patterns: '-e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"' + test_variables: '-e NETWORK' + network: 'Mainnet' # Test reconfiguring the docker image for testnet. test-configuration-file-testnet: - name: Test testnet Zebra CD Docker config file - timeout-minutes: 15 - runs-on: ubuntu-latest + name: Test CD testnet Docker config file needs: build - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - # Make sure Zebra can sync the genesis block on testnet - - name: Run tests using a testnet config - shell: /usr/bin/bash -exo pipefail {0} - run: | - docker pull ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - docker run --env "NETWORK=Testnet" --detach --name testnet-conf-tests -t ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - # Use a subshell to handle the broken pipe error gracefully - ( - trap "" PIPE; - docker logs \ - --tail all \ - --follow \ - testnet-conf-tests | \ - tee --output-error=exit /dev/stderr | \ - grep --max-count=1 --extended-regexp --color=always \ - -e "net.*=.*Test.*estimated progress to chain tip.*Genesis" \ - -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"; - ) || true - LOGS_EXIT_STATUS=$? - - docker stop testnet-conf-tests - - EXIT_STATUS=$(docker wait testnet-conf-tests || echo "Error retrieving exit status"); - echo "docker exit status: $EXIT_STATUS"; - - # If grep found the pattern, exit with the Docker container exit status - if [ $LOGS_EXIT_STATUS -eq 0 ]; then - exit $EXIT_STATUS; - fi - - # Handle other potential errors here - echo "An error occurred while processing the logs."; - exit 1; + # Make sure Zebra can sync the genesis block on testnet + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: 'testnet-conf' + docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} + grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"' + test_variables: '-e NETWORK' + network: 'Testnet' + + # Test that Zebra works using $ZEBRA_CONF_PATH config + test-zebra-conf-path: + name: Test CD custom Docker config file + needs: build + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: 'custom-conf' + docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} + grep_patterns: '-e "v1.0.0-rc.2.toml"' + test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"' + network: ${{ inputs.network || vars.ZCASH_NETWORK }} # Deploy Managed Instance Groups (MiGs) for Mainnet and Testnet, # with one node in the configured GCP region. @@ -225,14 +165,14 @@ jobs: # otherwise a new major version is deployed in a new MiG. # # Runs: - # - on every push/merge to the `main` branch + # - on every push to the `main` branch # - on every release, when it's published deploy-nodes: strategy: matrix: network: [Mainnet, Testnet] name: Deploy ${{ matrix.network }} nodes - needs: [ build, test-configuration-file, versioning ] + needs: [ build, versioning, test-configuration-file, test-zebra-conf-path ] runs-on: ubuntu-latest timeout-minutes: 60 permissions: @@ -334,7 +274,7 @@ jobs: # Note: this instances are not automatically replaced or deleted deploy-instance: name: Deploy single ${{ inputs.network }} instance - needs: [ build, test-configuration-file ] + needs: [ build, test-configuration-file, test-zebra-conf-path ] runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -401,7 +341,7 @@ jobs: failure-issue: name: Open or update issues for release failures # When a new job is added to this workflow, add it to this list. - needs: [ versioning, build, test-configuration-file, deploy-nodes, deploy-instance ] + needs: [ versioning, build, deploy-nodes, deploy-instance ] # Only open tickets for failed or cancelled jobs that are not coming from PRs. # (PR statuses are already reported in the PR jobs list, and checked by Mergify.) if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null) diff --git a/.github/workflows/delete-gcp-resources.yml b/.github/workflows/chore-delete-gcp-resources.yml similarity index 100% rename from .github/workflows/delete-gcp-resources.yml rename to .github/workflows/chore-delete-gcp-resources.yml diff --git a/.github/workflows/project-management.yml b/.github/workflows/chore-project-management.yml similarity index 100% rename from .github/workflows/project-management.yml rename to .github/workflows/chore-project-management.yml diff --git a/.github/workflows/build-crates-individually.patch.yml b/.github/workflows/ci-build-crates.patch.yml similarity index 97% rename from .github/workflows/build-crates-individually.patch.yml rename to .github/workflows/ci-build-crates.patch.yml index a4e69340fb5..b99adfc4cfe 100644 --- a/.github/workflows/build-crates-individually.patch.yml +++ b/.github/workflows/ci-build-crates.patch.yml @@ -14,7 +14,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/build-crates-individually.yml' + - '.github/workflows/ci-build-crates.yml' jobs: matrix: diff --git a/.github/workflows/build-crates-individually.yml b/.github/workflows/ci-build-crates.yml similarity index 98% rename from .github/workflows/build-crates-individually.yml rename to .github/workflows/ci-build-crates.yml index 33f8b1ef12c..e12f6031cb8 100644 --- a/.github/workflows/build-crates-individually.yml +++ b/.github/workflows/ci-build-crates.yml @@ -22,7 +22,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/build-crates-individually.yml' + - '.github/workflows/ci-build-crates.yml' pull_request: paths: # production code and test code @@ -34,7 +34,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/build-crates-individually.yml' + - '.github/workflows/ci-build-crates.yml' env: CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} diff --git a/.github/workflows/coverage.patch.yml b/.github/workflows/ci-coverage.patch.yml similarity index 93% rename from .github/workflows/coverage.patch.yml rename to .github/workflows/ci-coverage.patch.yml index 241f92e73e5..e2defe8ee7b 100644 --- a/.github/workflows/coverage.patch.yml +++ b/.github/workflows/ci-coverage.patch.yml @@ -12,7 +12,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' - 'codecov.yml' - - '.github/workflows/coverage.yml' + - '.github/workflows/ci-coverage.yml' jobs: coverage: diff --git a/.github/workflows/coverage.yml b/.github/workflows/ci-coverage.yml similarity index 97% rename from .github/workflows/coverage.yml rename to .github/workflows/ci-coverage.yml index b66944b38c5..0c913be1ad0 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/ci-coverage.yml @@ -27,7 +27,7 @@ on: - '**/clippy.toml' # workflow definitions - 'codecov.yml' - - '.github/workflows/coverage.yml' + - '.github/workflows/ci-coverage.yml' pull_request: paths: - '**/*.rs' @@ -38,7 +38,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' - 'codecov.yml' - - '.github/workflows/coverage.yml' + - '.github/workflows/ci-coverage.yml' env: CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/ci-integration-tests-gcp.patch.yml similarity index 67% rename from .github/workflows/continous-integration-docker.patch.yml rename to .github/workflows/ci-integration-tests-gcp.patch.yml index 4b9755519a7..f671429b5bf 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/ci-integration-tests-gcp.patch.yml @@ -1,4 +1,4 @@ -name: CI Docker +name: Integration Tests on GCP # These jobs *don't* depend on cached Google Cloud state disks, # so they can be skipped when the modified files make the actual workflow run. @@ -20,10 +20,10 @@ on: # workflow definitions - 'docker/**' - '.dockerignore' - - '.github/workflows/continous-integration-docker.yml' - - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/find-cached-disks.yml' - - '.github/workflows/build-docker-image.yml' + - '.github/workflows/ci-unit-tests-docker.yml' + - '.github/workflows/sub-deploy-integration-tests-gcp.yml' + - '.github/workflows/sub-find-cached-disks.yml' + - '.github/workflows/sub-build-docker-image.yml' jobs: # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable) @@ -39,48 +39,6 @@ jobs: steps: - run: 'echo "No build required"' - test-all: - name: Test all - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-all-getblocktemplate-rpcs: - name: Test all with getblocktemplate-rpcs feature - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-fake-activation-heights: - name: Test with fake activation heights - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-empty-sync: - name: Test checkpoint sync from empty state - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-lightwalletd-integration: - name: Test integration with lightwalletd - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-configuration-file: - name: Test Zebra default Docker config file - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-zebra-conf-path: - name: Test Zebra custom Docker config file - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - test-stateful-sync: name: Zebra checkpoint update / Run sync-past-checkpoint test runs-on: ubuntu-latest diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/ci-integration-tests-gcp.yml similarity index 71% rename from .github/workflows/continous-integration-docker.yml rename to .github/workflows/ci-integration-tests-gcp.yml index ee87f999ce0..1132762c210 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/ci-integration-tests-gcp.yml @@ -1,4 +1,4 @@ -name: CI Docker +name: Integration Tests on GCP # Ensures that only one workflow task will run at a time. Previous builds, if # already in process, will get cancelled. Only the latest commit will be allowed @@ -62,10 +62,10 @@ on: - '**/clippy.toml' # workflow definitions - 'docker/**' - - '.github/workflows/continous-integration-docker.yml' - - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/build-docker-image.yml' - - '.github/workflows/find-cached-disks.yml' + - '.github/workflows/ci-integration-tests-gcp.yml' + - '.github/workflows/sub-deploy-integration-tests-gcp.yml' + - '.github/workflows/sub-build-docker-image.yml' + - '.github/workflows/sub-find-cached-disks.yml' push: branches: @@ -86,10 +86,10 @@ on: # workflow definitions - 'docker/**' - '.dockerignore' - - '.github/workflows/continous-integration-docker.yml' - - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/find-cached-disks.yml' - - '.github/workflows/build-docker-image.yml' + - '.github/workflows/ci-integration-tests-gcp.yml' + - '.github/workflows/sub-deploy-integration-tests-gcp.yml' + - '.github/workflows/sub-find-cached-disks.yml' + - '.github/workflows/sub-build-docker-image.yml' jobs: # to also run a job on Mergify head branches, @@ -101,20 +101,20 @@ jobs: # The default network is mainnet unless a manually triggered workflow or repository variable # is configured differently. # - # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml + # The outputs for this job have the same names as the workflow outputs in sub-find-cached-disks.yml get-available-disks: name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }} - uses: ./.github/workflows/find-cached-disks.yml + uses: ./.github/workflows/sub-find-cached-disks.yml with: network: ${{ inputs.network || vars.ZCASH_NETWORK }} # Check if the cached state disks used by the tests are available for testnet. # - # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml + # The outputs for this job have the same names as the workflow outputs in sub-find-cached-disks.yml # Some outputs are ignored, because we don't run those jobs on testnet. get-available-disks-testnet: name: Check if cached state disks exist for testnet - uses: ./.github/workflows/find-cached-disks.yml + uses: ./.github/workflows/sub-find-cached-disks.yml with: network: 'Testnet' @@ -125,7 +125,7 @@ jobs: # testnet when running the image. build: name: Build CI Docker - uses: ./.github/workflows/build-docker-image.yml + uses: ./.github/workflows/sub-build-docker-image.yml with: dockerfile_path: ./docker/Dockerfile dockerfile_target: tests @@ -135,199 +135,6 @@ jobs: rust_lib_backtrace: full rust_log: info - # zebrad tests without cached state - - # TODO: make the non-cached-state tests use: - # network: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Run all the zebra tests, including tests that are ignored by default. - # Skips tests that need a cached state disk or a lightwalletd binary. - # - # - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step. - # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests. - # - # TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings - test-all: - name: Test all - runs-on: ubuntu-latest-xl - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails. - # - # If some tests hang, add "-- --nocapture" for just that test, or for all the tests. - - name: Run zebrad tests - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored - env: - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # zebrad tests without cached state with `getblocktemplate-rpcs` feature - # - # Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature. - test-all-getblocktemplate-rpcs: - name: Test all with getblocktemplate-rpcs feature - runs-on: ubuntu-latest-xl - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Run zebrad tests - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK --name zebrad-tests --tty -e ${{ inputs.network || vars.ZCASH_NETWORK }} ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored - env: - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Run state tests with fake activation heights. - # - # This test changes zebra-chain's activation heights, - # which can recompile all the Zebra crates, - # so we want its build products to be cached separately. - # - # Also, we don't want to accidentally use the fake heights in other tests. - # - # (The gRPC feature is a zebrad feature, so it isn't needed here.) - test-fake-activation-heights: - name: Test with fake activation heights - runs-on: ubuntu-latest - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Run tests with fake activation heights - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights - env: - TEST_FAKE_ACTIVATION_HEIGHTS: '1' - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. - # - # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.) - test-empty-sync: - name: Test checkpoint sync from empty state - runs-on: ubuntu-latest - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Run zebrad large sync tests - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_ - env: - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Test launching lightwalletd with an empty lightwalletd and Zebra state. - # - # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.) - test-lightwalletd-integration: - name: Test integration with lightwalletd - runs-on: ubuntu-latest - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Run tests with empty lightwalletd launch - run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration - env: - ZEBRA_TEST_LIGHTWALLETD: '1' - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Test that Zebra works using the default config with the latest Zebra version - test-configuration-file: - name: Test Zebra default Docker config file - timeout-minutes: 15 - runs-on: ubuntu-latest - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Run tests using the default config - run: | - set -ex - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start - EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; ) - docker stop default-conf-tests - docker logs default-conf-tests - exit "$EXIT_STATUS" - env: - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Test that Zebra works using the $ZEBRA_CONF_PATH config - test-zebra-conf-path: - name: Test Zebra custom Docker config file - timeout-minutes: 15 - runs-on: ubuntu-latest - needs: build - if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - steps: - - uses: r7kamura/rust-problem-matchers@v1.4.0 - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - with: - short-length: 7 - - - name: Run tests using the $ZEBRA_CONF_PATH - run: | - set -ex - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start - EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; ) - docker stop variable-conf-tests - docker logs variable-conf-tests - exit "$EXIT_STATUS" - env: - ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml' - NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # END TODO: make the non-cached-state tests use: - # network: ${{ inputs.network || vars.ZCASH_NETWORK }} - # zebrad cached checkpoint state tests # Regenerate mandatory checkpoint Zebra cached state disks. @@ -340,7 +147,7 @@ jobs: regenerate-stateful-disks: name: Zebra checkpoint needs: [ build, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }} with: app_name: zebrad @@ -369,7 +176,7 @@ jobs: test-stateful-sync: name: Zebra checkpoint update needs: [ regenerate-stateful-disks, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || needs.regenerate-stateful-disks.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebrad @@ -397,7 +204,7 @@ jobs: test-full-sync: name: Zebra tip needs: [ build, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }} with: app_name: zebrad @@ -440,7 +247,7 @@ jobs: test-update-sync: name: Zebra tip update needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebrad @@ -473,7 +280,7 @@ jobs: checkpoints-mainnet: name: Generate checkpoints mainnet needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebrad @@ -508,7 +315,7 @@ jobs: test-full-sync-testnet: name: Zebra tip on testnet needs: [ build, get-available-disks-testnet ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ (github.event_name == 'schedule' && vars.SCHEDULE_TESTNET_FULL_SYNC == 'true') || !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }} with: app_name: zebrad @@ -554,7 +361,7 @@ jobs: checkpoints-testnet: name: Generate checkpoints testnet needs: [ test-full-sync-testnet, get-available-disks-testnet ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || needs.test-full-sync-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebrad @@ -587,7 +394,7 @@ jobs: lightwalletd-full-sync: name: lightwalletd tip needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml # Currently the lightwalletd tests only work on Mainnet if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && (github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }} with: @@ -627,7 +434,7 @@ jobs: lightwalletd-update-sync: name: lightwalletd tip update needs: [ lightwalletd-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: lightwalletd @@ -659,7 +466,7 @@ jobs: lightwalletd-rpc-test: name: Zebra tip JSON-RPC needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: lightwalletd @@ -684,7 +491,7 @@ jobs: lightwalletd-transactions-test: name: lightwalletd tip send needs: [ lightwalletd-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: lightwalletd @@ -711,7 +518,7 @@ jobs: lightwalletd-grpc-test: name: lightwalletd GRPC tests needs: [ lightwalletd-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: lightwalletd @@ -742,7 +549,7 @@ jobs: get-block-template-test: name: get block template needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebrad @@ -768,7 +575,7 @@ jobs: submit-block-test: name: submit block needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml + uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} with: app_name: zebrad @@ -789,8 +596,8 @@ jobs: # # This list is for reliable tests that are run on the `main` branch. # Testnet jobs are not in this list, because we expect testnet to fail occasionally. - needs: [ regenerate-stateful-disks, test-full-sync, lightwalletd-full-sync, test-all, test-all-getblocktemplate-rpcs, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path, test-stateful-sync, test-update-sync, checkpoints-mainnet, lightwalletd-update-sync, lightwalletd-rpc-test, lightwalletd-transactions-test, lightwalletd-grpc-test, get-block-template-test, submit-block-test ] - # Only open tickets for failed or cancelled jobs that are not coming from PRs. + needs: [ regenerate-stateful-disks, test-full-sync, lightwalletd-full-sync, test-stateful-sync, test-update-sync, checkpoints-mainnet, lightwalletd-update-sync, lightwalletd-rpc-test, lightwalletd-transactions-test, lightwalletd-grpc-test, get-block-template-test, submit-block-test ] + # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. # (PR statuses are already reported in the PR jobs list, and checked by Mergify.) if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null) runs-on: ubuntu-latest diff --git a/.github/workflows/lint.patch.yml b/.github/workflows/ci-lint.patch.yml similarity index 100% rename from .github/workflows/lint.patch.yml rename to .github/workflows/ci-lint.patch.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/ci-lint.yml similarity index 99% rename from .github/workflows/lint.yml rename to .github/workflows/ci-lint.yml index ee4ba696d31..1e22a99a0f2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/ci-lint.yml @@ -45,7 +45,7 @@ jobs: **/Cargo.lock clippy.toml .cargo/config.toml - .github/workflows/lint.yml + .github/workflows/ci-lint.yml - name: Workflow files id: changed-files-workflows diff --git a/.github/workflows/ci-unit-tests-docker.patch.yml b/.github/workflows/ci-unit-tests-docker.patch.yml new file mode 100644 index 00000000000..e3e629fb490 --- /dev/null +++ b/.github/workflows/ci-unit-tests-docker.patch.yml @@ -0,0 +1,82 @@ +name: Docker Unit Tests + +# These jobs *don't* depend on cached Google Cloud state disks, +# so they can be skipped when the modified files make the actual workflow run. +on: + pull_request: + paths-ignore: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # test data snapshots + - '**/*.snap' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/ci-unit-tests-docker.yml' + - '.github/workflows/sub-deploy-integration-tests-gcp.yml' + - '.github/workflows/sub-find-cached-disks.yml' + - '.github/workflows/sub-build-docker-image.yml' + +jobs: + # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable) + get-available-disks: + name: Check if cached state disks exist for Mainnet / Check if cached state disks exist + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + build: + name: Build CI Docker / Build images + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-all: + name: Test all + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-all-getblocktemplate-rpcs: + name: Test all with getblocktemplate-rpcs feature + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-fake-activation-heights: + name: Test with fake activation heights + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-empty-sync: + name: Test checkpoint sync from empty state + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-lightwalletd-integration: + name: Test integration with lightwalletd + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-configuration-file: + name: Test CI default Docker config file / Test default-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-zebra-conf-path: + name: Test CI custom Docker config file / Test custom-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/ci-unit-tests-docker.yml new file mode 100644 index 00000000000..fcd8e74d41f --- /dev/null +++ b/.github/workflows/ci-unit-tests-docker.yml @@ -0,0 +1,269 @@ +name: Docker Unit Tests + +# Ensures that only one workflow task will run at a time. Previous builds, if +# already in process, will get cancelled. Only the latest commit will be allowed +# to run, cancelling any workflows in between +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + inputs: + network: + default: 'Mainnet' + description: 'Network to deploy: Mainnet or Testnet' + required: true + no_cache: + description: 'Disable the Docker cache for this build' + required: false + type: boolean + default: false + + pull_request: + paths: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # test data snapshots + - '**/*.snap' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.github/workflows/ci-unit-tests-docker.yml' + - '.github/workflows/sub-deploy-integration-tests-gcp.yml' + - '.github/workflows/sub-build-docker-image.yml' + - '.github/workflows/sub-find-cached-disks.yml' + + push: + branches: + - main + paths: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # test data snapshots + - '**/*.snap' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/ci-unit-tests-docker.yml' + - '.github/workflows/sub-deploy-integration-tests-gcp.yml' + - '.github/workflows/sub-find-cached-disks.yml' + - '.github/workflows/sub-build-docker-image.yml' + +jobs: + # Build the docker image used by the tests. + # + # The default network in the Zebra config in the image is mainnet, unless a manually triggered + # workflow or repository variable is configured differently. Testnet jobs change that config to + # testnet when running the image. + build: + name: Build CI Docker + uses: ./.github/workflows/sub-build-docker-image.yml + with: + dockerfile_path: ./docker/Dockerfile + dockerfile_target: tests + image_name: ${{ vars.CI_IMAGE_NAME }} + no_cache: ${{ inputs.no_cache || false }} + rust_backtrace: full + rust_lib_backtrace: full + rust_log: info + + # Run all the zebra tests, including tests that are ignored by default. + # + # - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step. + # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests. + # + # TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings + test-all: + name: Test all + runs-on: ubuntu-latest-xl + needs: build + steps: + - uses: r7kamura/rust-problem-matchers@v1.4.0 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails. + # + # If some tests hang, add "-- --nocapture" for just that test, or for all the tests. + - name: Run zebrad tests + run: | + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored + env: + NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # zebrad tests without cached state with `getblocktemplate-rpcs` feature + # + # Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature. + test-all-getblocktemplate-rpcs: + name: Test all with getblocktemplate-rpcs feature + runs-on: ubuntu-latest-xl + needs: build + steps: + - uses: r7kamura/rust-problem-matchers@v1.4.0 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Run zebrad tests + run: | + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker run -e NETWORK --name zebrad-tests --tty -e ${{ inputs.network || vars.ZCASH_NETWORK }} ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored + env: + NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # Run state tests with fake activation heights. + # + # This test changes zebra-chain's activation heights, + # which can recompile all the Zebra crates, + # so we want its build products to be cached separately. + # + # Also, we don't want to accidentally use the fake heights in other tests. + # + # (The gRPC feature is a zebrad feature, so it isn't needed here.) + test-fake-activation-heights: + name: Test with fake activation heights + runs-on: ubuntu-latest + needs: build + steps: + - uses: r7kamura/rust-problem-matchers@v1.4.0 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Run tests with fake activation heights + run: | + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights + env: + TEST_FAKE_ACTIVATION_HEIGHTS: '1' + NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. + # + # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.) + test-empty-sync: + name: Test checkpoint sync from empty state + runs-on: ubuntu-latest + needs: build + steps: + - uses: r7kamura/rust-problem-matchers@v1.4.0 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Run zebrad large sync tests + run: | + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_ + env: + NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # Test launching lightwalletd with an empty lightwalletd and Zebra state. + # + # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.) + test-lightwalletd-integration: + name: Test integration with lightwalletd + runs-on: ubuntu-latest + needs: build + steps: + - uses: r7kamura/rust-problem-matchers@v1.4.0 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Run tests with empty lightwalletd launch + run: | + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration + env: + ZEBRA_TEST_LIGHTWALLETD: '1' + NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # Test that Zebra works using the default config with the latest Zebra version. + test-configuration-file: + name: Test CI default Docker config file + needs: build + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: 'default-conf' + docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + grep_patterns: '-e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"' + test_variables: '-e NETWORK' + network: 'Mainnet' + + # Test reconfiguring the the docker image for tesnet. + test-configuration-file-testnet: + name: Test CI testnet Docker config file + needs: build + # Make sure Zebra can sync the genesis block on testnet + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: 'testnet-conf' + docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"' + # TODO: improve the entrypoint to avoid using `ENTRYPOINT_FEATURES=""` + test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="/etc/zebrad/zebrad.toml" -e ENTRYPOINT_FEATURES=""' + network: 'Testnet' + + # Test that Zebra works using $ZEBRA_CONF_PATH config + test-zebra-conf-path: + name: Test CI custom Docker config file + needs: build + uses: ./.github/workflows/sub-test-zebra-config.yml + with: + test_id: 'custom-conf' + docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + grep_patterns: '-e "v1.0.0-rc.2.toml"' + test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"' + network: ${{ inputs.network || vars.ZCASH_NETWORK }} + + failure-issue: + name: Open or update issues for main branch failures + # When a new test is added to this workflow, add it to this list. + # + # This list is for reliable tests that are run on the `main` branch. + # Testnet jobs are not in this list, because we expect testnet to fail occasionally. + needs: [ test-all, test-all-getblocktemplate-rpcs, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path ] + # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. + # (PR statuses are already reported in the PR jobs list, and checked by Mergify.) + # TODO: if a job times out, we want to create a ticket. Does failure() do that? Or do we need cancelled()? + if: failure() && github.event.pull_request == null + runs-on: ubuntu-latest + steps: + - uses: jayqi/failed-build-issue-action@v1 + with: + title-template: "{{refname}} branch CI failed: {{eventName}} in {{workflow}}" + # New failures open an issue with this label. + # TODO: do we want a different label for each workflow, or each kind of workflow? + label-name: S-ci-fail-auto-issue + # If there is already an open issue with this label, any failures become comments on that issue. + always-create-new-issue: false + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/continous-integration-os.patch.yml b/.github/workflows/ci-unit-tests-os.patch.yml similarity index 96% rename from .github/workflows/continous-integration-os.patch.yml rename to .github/workflows/ci-unit-tests-os.patch.yml index fe81951024a..2d126cba054 100644 --- a/.github/workflows/continous-integration-os.patch.yml +++ b/.github/workflows/ci-unit-tests-os.patch.yml @@ -1,4 +1,4 @@ -name: CI OSes +name: Multi-OS Unit Tests on: pull_request: @@ -11,7 +11,7 @@ on: - '**/deny.toml' - '.cargo/config.toml' - '**/clippy.toml' - - '.github/workflows/continous-integration-os.yml' + - '.github/workflows/ci-unit-tests-os.yml' jobs: test: diff --git a/.github/workflows/continous-integration-os.yml b/.github/workflows/ci-unit-tests-os.yml similarity index 97% rename from .github/workflows/continous-integration-os.yml rename to .github/workflows/ci-unit-tests-os.yml index 63e7006414b..00594afacd5 100644 --- a/.github/workflows/continous-integration-os.yml +++ b/.github/workflows/ci-unit-tests-os.yml @@ -1,4 +1,4 @@ -name: CI OSes +name: Multi-OS Unit Tests # Ensures that only one workflow task will run at a time. Previous builds, if # already in process, will get cancelled. Only the latest commit will be allowed @@ -12,14 +12,12 @@ on: # we build Rust and Zcash parameter caches on main, # so they can be shared by all branches: # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache - push: - branches: - - main + + pull_request: paths: - # production code and test code + # code and tests - '**/*.rs' - # hard-coded checkpoints - # TODO: skip proptest regressions? + # hard-coded checkpoints and proptest regressions - '**/*.txt' # test data snapshots - '**/*.snap' @@ -31,12 +29,16 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/continous-integration-os.yml' - pull_request: + - '.github/workflows/ci-unit-tests-os.yml' + + push: + branches: + - main paths: - # code and tests + # production code and test code - '**/*.rs' - # hard-coded checkpoints and proptest regressions + # hard-coded checkpoints + # TODO: skip proptest regressions? - '**/*.txt' # test data snapshots - '**/*.snap' @@ -48,7 +50,8 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/continous-integration-os.yml' + - '.github/workflows/ci-unit-tests-os.yml' + - '.github/workflows/sub-build-docker-image.yml' env: CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} @@ -58,6 +61,9 @@ env: COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }} jobs: + ######################################## + ### Build and test Zebra on all OSes ### + ######################################## test: name: Test ${{ matrix.rust }} on ${{ matrix.os }}${{ matrix.features }} # The large timeout is to accommodate: @@ -233,7 +239,6 @@ jobs: - name: Install last version of Protoc uses: arduino/setup-protoc@v2.1.0 with: - # TODO: increase to latest version after https://github.com/arduino/setup-protoc/issues/33 is fixed version: '23.x' repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/continous-delivery.patch.yml b/.github/workflows/continous-delivery.patch.yml deleted file mode 100644 index f51ef601468..00000000000 --- a/.github/workflows/continous-delivery.patch.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: CD - -on: - # Only patch the Docker image test jobs - pull_request: - paths-ignore: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/continous-delivery.yml' - - '.github/workflows/find-cached-disks.yml' - - -jobs: - build: - name: Build CD Docker / Build images - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-configuration-file: - name: Test Zebra CD Docker config file - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' diff --git a/.github/workflows/continous-integration-docker.patch-always.yml b/.github/workflows/continous-integration-docker.patch-always.yml deleted file mode 100644 index e05c97f9d24..00000000000 --- a/.github/workflows/continous-integration-docker.patch-always.yml +++ /dev/null @@ -1,26 +0,0 @@ -# These jobs can be skipped based on cached Google Cloud state disks, -# and some of them just run on the `main` branch, -# so the patch jobs always need to run on every PR. -name: CI Docker - -on: - pull_request: - -jobs: - regenerate-stateful-disks: - name: Zebra checkpoint / Run sync-to-checkpoint test - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-full-sync: - name: Zebra tip / Run full-sync test - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - lightwalletd-full-sync: - name: lightwalletd tip / Run lwd-full-sync test - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' diff --git a/.github/workflows/docs.patch.yml b/.github/workflows/docs-deploy-firebase.patch.yml similarity index 94% rename from .github/workflows/docs.patch.yml rename to .github/workflows/docs-deploy-firebase.patch.yml index 0c3c94aca03..04b4bdc46d4 100644 --- a/.github/workflows/docs.patch.yml +++ b/.github/workflows/docs-deploy-firebase.patch.yml @@ -18,7 +18,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/docs.yml' + - '.github/workflows/docs-deploy-firebase.yml' jobs: build-docs-book: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs-deploy-firebase.yml similarity index 99% rename from .github/workflows/docs.yml rename to .github/workflows/docs-deploy-firebase.yml index 9d22e44cf13..078c2e6a1fe 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs-deploy-firebase.yml @@ -26,7 +26,7 @@ on: - '.cargo/config.toml' - '**/clippy.toml' # workflow definitions - - '.github/workflows/docs.yml' + - '.github/workflows/docs-deploy-firebase.yml' pull_request: branches: diff --git a/.github/workflows/dockerhub-description.yml b/.github/workflows/docs-dockerhub-description.yml similarity index 100% rename from .github/workflows/dockerhub-description.yml rename to .github/workflows/docs-dockerhub-description.yml diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/manual-zcashd-deploy.yml similarity index 100% rename from .github/workflows/zcashd-manual-deploy.yml rename to .github/workflows/manual-zcashd-deploy.yml diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 2fe2243e0b5..50c28acb7f8 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -1,5 +1,5 @@ # This workflow is meant to trigger a build of Docker binaries when a release -# is published, it uses the existing `build-docker-image.yml` workflow +# is published, it uses the existing `sub-build-docker-image.yml` workflow # # We use a separate action as we might want to trigger this under # different circumstances than a Continuous Deployment, for example. @@ -21,7 +21,7 @@ jobs: # The image will be named `zebra:` build: name: Build Release Docker - uses: ./.github/workflows/build-docker-image.yml + uses: ./.github/workflows/sub-build-docker-image.yml with: dockerfile_path: ./docker/Dockerfile dockerfile_target: runtime @@ -33,7 +33,7 @@ jobs: # The image will be named `zebra:.experimental` build-mining-testnet: name: Build Release Testnet Mining Docker - uses: ./.github/workflows/build-docker-image.yml + uses: ./.github/workflows/sub-build-docker-image.yml with: dockerfile_path: ./docker/Dockerfile dockerfile_target: runtime diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index d3987e0a048..39e1ba7bde2 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -37,6 +37,7 @@ jobs: - uses: release-drafter/release-drafter@v5 with: config-name: release-drafter.yml + commitish: main #disable-autolabeler: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml similarity index 100% rename from .github/workflows/build-docker-image.yml rename to .github/workflows/sub-build-docker-image.yml diff --git a/.github/workflows/zcash-lightwalletd.patch.yml b/.github/workflows/sub-build-lightwalletd.patch.yml similarity index 85% rename from .github/workflows/zcash-lightwalletd.patch.yml rename to .github/workflows/sub-build-lightwalletd.patch.yml index 1c4413c2975..c126aec0ffb 100644 --- a/.github/workflows/zcash-lightwalletd.patch.yml +++ b/.github/workflows/sub-build-lightwalletd.patch.yml @@ -1,4 +1,4 @@ -name: zcash-lightwalletd +name: Build lightwalletd # When the real job doesn't run because the files aren't changed, # run a fake CI job to satisfy the branch protection rules. @@ -10,7 +10,7 @@ on: - 'zebrad/src/config.rs' - 'zebrad/src/commands/start.rs' - 'docker/zcash-lightwalletd/Dockerfile' - - '.github/workflows/zcash-lightwalletd.yml' + - '.github/workflows/sub-build-lightwalletd.yml' jobs: build: diff --git a/.github/workflows/zcash-lightwalletd.yml b/.github/workflows/sub-build-lightwalletd.yml similarity index 97% rename from .github/workflows/zcash-lightwalletd.yml rename to .github/workflows/sub-build-lightwalletd.yml index 7ef1786e1f6..adce75ed7b7 100644 --- a/.github/workflows/zcash-lightwalletd.yml +++ b/.github/workflows/sub-build-lightwalletd.yml @@ -1,6 +1,6 @@ # TODO: we should stop using this build approach with lightwalletd and move to using our # reusable workflow to building all the docker images of our repo -name: zcash-lightwalletd +name: Build lightwalletd # Ensures that only one workflow task will run at a time. Previous builds, if # already in process, will get cancelled. Only the latest commit will be allowed @@ -29,7 +29,7 @@ on: - 'zebrad/src/commands/start.rs' # these workflow definitions actually change the docker image - 'docker/zcash-lightwalletd/Dockerfile' - - '.github/workflows/zcash-lightwalletd.yml' + - '.github/workflows/sub-build-lightwalletd.yml' # Update the lightwalletd image when each related PR changes pull_request: @@ -42,7 +42,7 @@ on: - 'zebrad/src/commands/start.rs' # these workflow definitions actually change the docker image - 'docker/zcash-lightwalletd/Dockerfile' - - '.github/workflows/zcash-lightwalletd.yml' + - '.github/workflows/sub-build-lightwalletd.yml' env: IMAGE_NAME: lightwalletd diff --git a/.github/workflows/zcash-params.yml b/.github/workflows/sub-build-zcash-params.yml similarity index 86% rename from .github/workflows/zcash-params.yml rename to .github/workflows/sub-build-zcash-params.yml index 28bcea9a424..ee08b6fd3c2 100644 --- a/.github/workflows/zcash-params.yml +++ b/.github/workflows/sub-build-zcash-params.yml @@ -1,4 +1,4 @@ -name: zcash-params +name: Build zcash-params # Ensures that only one workflow task will run at a time. Previous deployments, if # already in process, won't get cancelled. Instead, we let the first to complete @@ -28,13 +28,13 @@ on: # workflow definitions - 'docker/zcash-params/Dockerfile' - '.dockerignore' - - '.github/workflows/zcash-params.yml' - - '.github/workflows/build-docker-image.yml' + - '.github/workflows/sub-build-zcash-params.yml' + - '.github/workflows/sub-build-docker-image.yml' jobs: build: name: Build Zcash Params Docker - uses: ./.github/workflows/build-docker-image.yml + uses: ./.github/workflows/sub-build-docker-image.yml with: dockerfile_path: ./docker/zcash-params/Dockerfile dockerfile_target: release diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/sub-deploy-integration-tests-gcp.yml similarity index 99% rename from .github/workflows/deploy-gcp-tests.yml rename to .github/workflows/sub-deploy-integration-tests-gcp.yml index cd0076c2e19..e718eaa43af 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/sub-deploy-integration-tests-gcp.yml @@ -1,4 +1,4 @@ -name: Deploy GCP tests +name: Deploy Tests to GCP on: workflow_call: @@ -311,7 +311,7 @@ jobs: # Passes the disk name to subsequent steps using $CACHED_DISK_NAME env variable # Passes the state version to subsequent steps using $STATE_VERSION env variable # - # TODO: move this script into a file, and call it from find-cached-disks.yml as well. + # TODO: move this script into a file, and call it from sub-find-cached-disks.yml as well. - name: Find ${{ inputs.test_id }} cached state disk id: get-disk-name run: | @@ -414,7 +414,7 @@ jobs: # - /var/cache/zebrad-cache -> ${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} -> $ZEBRA_CACHED_STATE_DIR # # This path must match the variable used by the tests in Rust, which are also set in - # `continous-integration-docker.yml` to be able to run this tests. + # `ci-unit-tests-docker.yml` to be able to run this tests. # # Although we're mounting the disk root, Zebra will only respect the values from # $ZEBRA_CACHED_STATE_DIR. The inputs like ${{ inputs.zebra_state_dir }} are only used @@ -485,7 +485,7 @@ jobs: # delete the whole cache directory.) # # This paths must match the variables used by the tests in Rust, which are also set in - # `continous-integration-docker.yml` to be able to run this tests. + # `ci-unit-tests-docker.yml` to be able to run this tests. # # Although we're mounting the disk root to both directories, Zebra and Lightwalletd # will only respect the values from $ZEBRA_CACHED_STATE_DIR and $LIGHTWALLETD_DATA_DIR, diff --git a/.github/workflows/find-cached-disks.yml b/.github/workflows/sub-find-cached-disks.yml similarity index 100% rename from .github/workflows/find-cached-disks.yml rename to .github/workflows/sub-find-cached-disks.yml diff --git a/.github/workflows/sub-test-zebra-config.yml b/.github/workflows/sub-test-zebra-config.yml new file mode 100644 index 00000000000..d8e856f0748 --- /dev/null +++ b/.github/workflows/sub-test-zebra-config.yml @@ -0,0 +1,79 @@ +name: Test Zebra Config Files + +on: + workflow_call: + inputs: + # Status and logging + test_id: + required: true + type: string + description: 'Unique identifier for the test' + grep_patterns: + required: true + type: string + description: 'Patterns to grep for in the logs' + + # Test selection and parameters + docker_image: + required: true + type: string + description: 'Docker image to test' + test_variables: + required: true + type: string + description: 'Environmental variables used to select and configure the test' + network: + required: false + type: string + default: Mainnet + description: 'Zcash network to test against' + +jobs: + test-docker-config: + name: Test ${{ inputs.test_id }} in Docker + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - uses: r7kamura/rust-problem-matchers@v1.4.0 + + - name: Run ${{ inputs.test_id }} test + run: | + docker pull ${{ inputs.docker_image }} + docker run ${{ inputs.test_variables }} --detach --name ${{ inputs.test_id }} -t ${{ inputs.docker_image }} zebrad start + # Use a subshell to handle the broken pipe error gracefully + ( + trap "" PIPE; + docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + ${{ inputs.grep_patterns }} + ) || true + LOGS_EXIT_STATUS=$? + + docker stop ${{ inputs.test_id }} + + EXIT_STATUS=$(docker wait ${{ inputs.test_id }} || echo "Error retrieving exit status"); + echo "docker exit status: $EXIT_STATUS"; + + # If grep found the pattern, exit with the Docker container exit status + if [ $LOGS_EXIT_STATUS -eq 0 ]; then + exit $EXIT_STATUS; + fi + + # Handle other potential errors here + echo "An error occurred while processing the logs."; + exit 1; + env: + NETWORK: '${{ inputs.network }}' diff --git a/book/src/dev/continuous-integration.md b/book/src/dev/continuous-integration.md index 3a7726fb3e0..895085f6395 100644 --- a/book/src/dev/continuous-integration.md +++ b/book/src/dev/continuous-integration.md @@ -90,7 +90,7 @@ This means that the entire workflow must be re-run when a single test fails. 1. Look for the earliest job that failed, and find the earliest failure. For example, this failure doesn't tell us what actually went wrong: -> Error: The template is not valid. ZcashFoundation/zebra/.github/workflows/build-docker-image.yml@8bbc5b21c97fafc83b70fbe7f3b5e9d0ffa19593 (Line: 52, Col: 19): Error reading JToken from JsonReader. Path '', line 0, position 0. +> Error: The template is not valid. ZcashFoundation/zebra/.github/workflows/sub-build-docker-image.yml@8bbc5b21c97fafc83b70fbe7f3b5e9d0ffa19593 (Line: 52, Col: 19): Error reading JToken from JsonReader. Path '', line 0, position 0. https://github.com/ZcashFoundation/zebra/runs/8181760421?check_suite_focus=true#step:41:4 diff --git a/docker/Dockerfile b/docker/Dockerfile index 2b4c770e328..f27ff99fa71 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ # # Build zebrad with these features # Keep these in sync with: -# https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L37 +# https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/sub-build-docker-image.yml#L37 ARG FEATURES="default-release-binaries" ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints" @@ -190,8 +190,8 @@ ARG FEATURES ENV FEATURES=${FEATURES} # Path and name of the config file -ENV ZEBRA_CONF_DIR=/etc/zebrad -ENV ZEBRA_CONF_FILE=zebrad.toml +ENV ZEBRA_CONF_DIR=${ZEBRA_CONF_DIR:-/etc/zebrad} +ENV ZEBRA_CONF_FILE=${ZEBRA_CONF_FILE:-zebrad.toml} # Expose configured ports EXPOSE 8233 18233 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 66b8f586586..79dd0c3bbfa 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -79,7 +79,7 @@ fi : "${ENTRYPOINT_FEATURES:=}" # Configuration file path -if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]]; then +if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]] && [[ -z "${ZEBRA_CONF_PATH}" ]]; then ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" fi @@ -90,8 +90,7 @@ fi # Users have to opt-in to additional functionality by setting environmental variables. if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then # Create the conf path and file - mkdir -p "${ZEBRA_CONF_DIR}" || { echo "Error creating directory ${ZEBRA_CONF_DIR}"; exit 1; } - touch "${ZEBRA_CONF_PATH}" || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } + (mkdir -p "$(dirname "${ZEBRA_CONF_PATH}")" && touch "${ZEBRA_CONF_PATH}") || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } # Populate the conf file cat < "${ZEBRA_CONF_PATH}" [network] From 01168c8571a517318bca37052ff16d7cc1689c68 Mon Sep 17 00:00:00 2001 From: Arya Date: Wed, 18 Oct 2023 02:16:29 -0400 Subject: [PATCH 2/2] change(state): Set upper bound when reading from deleting column family tx_loc_by_transparent_addr_loc (#7732) * Uses range_iter in address_transaction_locations * Uses range_iter in address_transaction_locations * uses u16::MAX instead of usize::MAX * Moves limit code into method * adds allow(dead_code) * Simplifies address_iterator_range * Moves test state init out of loop * Updates docs --- zebra-rpc/src/methods/tests/vectors.rs | 32 +++++++++++---- .../finalized_state/disk_format/block.rs | 1 - .../disk_format/transparent.rs | 40 +++++++++--------- .../finalized_state/zebra_db/transparent.rs | 41 +++---------------- 4 files changed, 51 insertions(+), 63 deletions(-) diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 769d561897b..68f08c184b1 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -16,6 +16,7 @@ use zebra_chain::{ }; use zebra_node_services::BoxError; +use zebra_state::{LatestChainTip, ReadStateService}; use zebra_test::mock_service::MockService; use super::super::*; @@ -674,19 +675,36 @@ async fn rpc_getaddresstxids_response() { .address(network) .unwrap(); + // Create a populated state service + let (_state, read_state, latest_chain_tip, _chain_tip_change) = + zebra_state::populated_state(blocks.to_owned(), network).await; + if network == Mainnet { // Exhaustively test possible block ranges for mainnet. // // TODO: if it takes too long on slower machines, turn this into a proptest with 10-20 cases for start in 1..=10 { for end in start..=10 { - rpc_getaddresstxids_response_with(network, start..=end, &blocks, &address) - .await; + rpc_getaddresstxids_response_with( + network, + start..=end, + &address, + &read_state, + &latest_chain_tip, + ) + .await; } } } else { // Just test the full range for testnet. - rpc_getaddresstxids_response_with(network, 1..=10, &blocks, &address).await; + rpc_getaddresstxids_response_with( + network, + 1..=10, + &address, + &read_state, + &latest_chain_tip, + ) + .await; } } } @@ -694,13 +712,11 @@ async fn rpc_getaddresstxids_response() { async fn rpc_getaddresstxids_response_with( network: Network, range: RangeInclusive, - blocks: &[Arc], address: &transparent::Address, + read_state: &ReadStateService, + latest_chain_tip: &LatestChainTip, ) { let mut mempool: MockService<_, _, _, BoxError> = MockService::build().for_unit_tests(); - // Create a populated state service - let (_state, read_state, latest_chain_tip, _chain_tip_change) = - zebra_state::populated_state(blocks.to_owned(), network).await; let (rpc, rpc_tx_queue_task_handle) = RpcImpl::new( "RPC test", @@ -710,7 +726,7 @@ async fn rpc_getaddresstxids_response_with( true, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), - latest_chain_tip, + latest_chain_tip.clone(), ); // call the method with valid arguments diff --git a/zebra-state/src/service/finalized_state/disk_format/block.rs b/zebra-state/src/service/finalized_state/disk_format/block.rs index 84efd60b016..03a7f648053 100644 --- a/zebra-state/src/service/finalized_state/disk_format/block.rs +++ b/zebra-state/src/service/finalized_state/disk_format/block.rs @@ -132,7 +132,6 @@ pub struct TransactionLocation { impl TransactionLocation { /// Creates a transaction location from a block height and transaction index. - #[allow(dead_code)] pub fn from_index(height: Height, transaction_index: u16) -> TransactionLocation { TransactionLocation { height, diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index ace95b04b0f..d3fb01c390f 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -416,36 +416,37 @@ impl AddressTransaction { } } - /// Create an [`AddressTransaction`] which starts iteration for the supplied - /// address. Starts at the first UTXO, or at the `query_start` height, - /// whichever is greater. + /// Create a range of [`AddressTransaction`]s which starts iteration for the supplied + /// address. Starts at the first UTXO, or at the `query` start height, whichever is greater. + /// Ends at the maximum possible transaction index for the end height. /// - /// Used to look up the first transaction with - /// [`ReadDisk::zs_next_key_value_from`][1]. + /// Used to look up transactions with [`DiskDb::zs_range_iter`][1]. /// - /// The transaction location might be invalid, if it is based on the - /// `query_start` height. But this is not an issue, since - /// [`ReadDisk::zs_next_key_value_from`][1] will fetch the next existing - /// (valid) value. + /// The transaction locations in the: + /// - start bound might be invalid, if it is based on the `query` start height. + /// - end bound will always be invalid. /// - /// [1]: super::super::disk_db::ReadDisk::zs_next_key_value_from - pub fn address_iterator_start( + /// But this is not an issue, since [`DiskDb::zs_range_iter`][1] will fetch all existing + /// (valid) values in the range. + /// + /// [1]: super::super::disk_db::DiskDb + pub fn address_iterator_range( address_location: AddressLocation, - query_start: Height, - ) -> AddressTransaction { + query: std::ops::RangeInclusive, + ) -> std::ops::RangeInclusive { // Iterating from the lowest possible transaction location gets us the first transaction. // // The address location is the output location of the first UTXO sent to the address, // and addresses can not spend funds until they receive their first UTXO. let first_utxo_location = address_location.transaction_location(); - // Iterating from the start height filters out transactions that aren't needed. - let query_start_location = TransactionLocation::from_usize(query_start, 0); + // Iterating from the start height to the end height filters out transactions that aren't needed. + let query_start_location = TransactionLocation::from_index(*query.start(), 0); + let query_end_location = TransactionLocation::from_index(*query.end(), u16::MAX); - AddressTransaction { - address_location, - transaction_location: max(first_utxo_location, query_start_location), - } + let addr_tx = |tx_loc| AddressTransaction::new(address_location, tx_loc); + + addr_tx(max(first_utxo_location, query_start_location))..=addr_tx(query_end_location) } /// Update the transaction location to the next possible transaction for the @@ -457,6 +458,7 @@ impl AddressTransaction { /// existing (valid) value. /// /// [1]: super::super::disk_db::ReadDisk::zs_next_key_value_from + #[allow(dead_code)] pub fn address_iterator_next(&mut self) { // Iterating from the next possible output location gets us the next output, // even if it is in a later block or transaction. diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index 2f80751687f..2e8d6c3980a 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -235,44 +235,15 @@ impl ZebraDb { let tx_loc_by_transparent_addr_loc = self.db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap(); - // Manually fetch the entire addresses' transaction locations - let mut addr_transactions = BTreeSet::new(); - // A potentially invalid key representing the first UTXO send to the address, // or the query start height. - let mut transaction_location = AddressTransaction::address_iterator_start( - address_location, - *query_height_range.start(), - ); - - loop { - // Seek to a valid entry for this address, or the first entry for the next address - transaction_location = match self - .db - .zs_next_key_value_from(&tx_loc_by_transparent_addr_loc, &transaction_location) - { - Some((transaction_location, ())) => transaction_location, - // We're finished with the final address in the column family - None => break, - }; - - // We found the next address, so we're finished with this address - if transaction_location.address_location() != address_location { - break; - } - - // We're past the end height, so we're finished with this query - if transaction_location.transaction_location().height > *query_height_range.end() { - break; - } + let transaction_location_range = + AddressTransaction::address_iterator_range(address_location, query_height_range); - addr_transactions.insert(transaction_location); - - // A potentially invalid key representing the next possible output - transaction_location.address_iterator_next(); - } - - addr_transactions + self.db + .zs_range_iter(&tx_loc_by_transparent_addr_loc, transaction_location_range) + .map(|(tx_loc, ())| tx_loc) + .collect() } // Address index queries