initialize-archiver #11
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: initialize-archiver | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| datasets: | |
| description: 'Comma-separated list of datasets to archive (e.g., "ferc2","ferc6").' | |
| default: "" | |
| required: true | |
| type: string | |
| sandbox: | |
| description: "Initialize a new archive on the Zenodo sandbox server?" | |
| default: true | |
| required: true | |
| type: boolean | |
| jobs: | |
| archive-run: | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| matrix: | |
| # Note that we can't pass global env variables to the matrix, so we manually reproduce the list of datasets here. | |
| dataset: ${{ fromJSON(format('[{0}]', inputs.datasets || '' )) }} | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set default GCP credentials | |
| id: gcloud-auth | |
| continue-on-error: true | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" | |
| service_account: "[email protected]" | |
| - name: Install Conda environment using mamba | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| environment-file: environment.yml | |
| cache-environment: true | |
| condarc: | | |
| channels: | |
| - conda-forge | |
| channel_priority: strict | |
| - name: Log the conda environment | |
| run: | | |
| conda info | |
| conda list | |
| conda config --show-sources | |
| conda config --show | |
| printenv | sort | |
| - name: Initialize production archive for ${{ matrix.dataset }} | |
| env: | |
| EPACEMS_API_KEY: ${{ secrets.EPACEMS_API_KEY }} | |
| ZENODO_TOKEN_UPLOAD: ${{ secrets.ZENODO_TOKEN_UPLOAD }} | |
| ZENODO_TOKEN_PUBLISH: ${{ secrets.ZENODO_TOKEN_PUBLISH }} | |
| if: ${{ inputs.sandbox == false }} | |
| run: | | |
| pudl_archiver --datasets ${{ matrix.dataset }} --initialize --summary-file ${{ matrix.dataset }}_run_summary.json --clobber-unchanged | |
| - name: Initialize sandbox archive for ${{ matrix.dataset }} | |
| env: | |
| ZENODO_SANDBOX_TOKEN_UPLOAD: ${{ secrets.ZENODO_SANDBOX_TOKEN_UPLOAD }} | |
| ZENODO_SANDBOX_TOKEN_PUBLISH: ${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} | |
| EPACEMS_API_KEY: ${{ secrets.EPACEMS_API_KEY }} | |
| if: ${{ inputs.sandbox == true }} | |
| run: | | |
| pudl_archiver --datasets ${{ matrix.dataset }} --initialize --sandbox --summary-file ${{ matrix.dataset }}_run_summary.json --clobber-unchanged | |
| - name: Upload run summaries | |
| if: always() | |
| id: upload_summaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: run-summaries-${{ matrix.dataset }} | |
| path: ${{ matrix.dataset }}_run_summary.json | |
| archive-notify: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - archive-run | |
| if: ${{ always() }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download summaries | |
| id: download | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: run-summaries-* | |
| merge-multiple: true | |
| - name: show summaries | |
| run: ls -R | |
| - name: Munge summaries together | |
| id: all_summaries | |
| run: | | |
| { | |
| echo "SLACK_PAYLOAD<<EOF" | |
| ./scripts/make_slack_notification_message.py --summary-files *_run_summary.json | tee slack-payload.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Post update to pudl-deployment | |
| uses: slackapi/[email protected] | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} | |
| payload: | | |
| text: "Created new archiver(s)." | |
| blocks: ${{ steps.all_summaries.outputs.SLACK_PAYLOAD }} | |
| channel: "C03FHB9N0PQ" |