Merge branch 'init-airtable-automation' of github.com:down-ballot-cli… #23
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: update-data | |
| on: | |
| push: | |
| tags: | |
| - "v20*" | |
| branches: | |
| - init-airtable-automation | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| env: | |
| API_KEY_GOOGLE_MAPS: ${{ secrets.API_KEY_GOOGLE_MAPS }} | |
| GITHUB_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule | |
| steps: | |
| - name: Use dev branch if running on a schedule | |
| if: ${{ (github.event_name == 'schedule') }} | |
| run: | | |
| echo "This action was triggered by a schedule." && echo "GITHUB_REF=dev" >> $GITHUB_ENV | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.GITHUB_REF }} | |
| - name: Who owns the workspace? | |
| run: ls -ld $GITHUB_WORKSPACE | |
| - uses: "google-github-actions/auth@v2" | |
| with: | |
| credentials_json: "${{ secrets.DGM_GITHUB_ACTION_CREDENTIALS }}" | |
| - name: Display env variables | |
| run: | | |
| echo "Workspace directory: $GITHUB_WORKSPACE" \ | |
| echo "Google credentials path: $GOOGLE_GHA_CREDS_PATH" \ | |
| # Give the dbcp user ownership of the workspace | |
| # So it can read and write files to the workspace | |
| - name: Give the dbcp user ownership of the workspace | |
| run: sudo chown -R 1000:1000 $GITHUB_WORKSPACE | |
| - name: Set up Docker Compose | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y docker-compose | |
| - name: Build and run Docker Compose services | |
| run: | | |
| docker-compose up -d | |
| - name: Run the archive | |
| env: | |
| AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} | |
| run: | | |
| make archive_all | |
| - name: Run full ETL | |
| if: ${{ success() }} | |
| run: | | |
| make all | |
| - name: Run all test | |
| run: | | |
| make test | |
| - name: Write settings.yaml | |
| id: write_settings | |
| if: ${{ success() }} | |
| run: | | |
| make save_settings | |
| # give permission to root | |
| - name: Give ownership of the workspace back to root | |
| run: sudo chown -R runner:docker $GITHUB_WORKSPACE | |
| # commit the settings file | |
| - uses: EndBug/add-and-commit@v9 | |
| id: commit_settings_file | |
| if: steps.write_settings.outcome == 'success' | |
| with: | |
| add: "src/dbcp/settings.yaml" | |
| author_name: "DBCP Bot" | |
| author_email: "[email protected]" | |
| message: "Update settings.yaml" | |
| push: true | |
| # Give the dbcp user ownership of the workspace | |
| # So it can read and write files to the workspace | |
| - name: Give the dbcp user ownership of the workspace | |
| run: sudo chown -R 1000:1000 $GITHUB_WORKSPACE | |
| # publish the outputs, grab the git sha of the commit step | |
| # TODO: REMOVE THE TRUE and build-ref!!! | |
| - name: Publish publish outputs | |
| if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || (env.GITHUB_REF == 'dev') || (true) | |
| run: | | |
| docker compose run --rm app python dbcp/cli.py publish-outputs \ | |
| -bq \ | |
| --build-ref 'sandbox' \ | |
| --code-git-sha ${{ github.sha }} \ | |
| --settings-file-git-sha ${{ steps.commit_settings_file.outputs.commit_long_sha }} \ | |
| --github-action-run-id ${{ github.run_id}} | |
| - name: Stop Docker Compose services | |
| if: always() | |
| run: | | |
| docker-compose down | |
| # The google-github-actions/auth step is run as runner:docker, | |
| # so we need to give the workspace back to runner:docker | |
| - name: Give ownership of the workspace back to root | |
| if: always() | |
| run: sudo chown -R runner:docker $GITHUB_WORKSPACE | |
| - name: Who owns the workspace? | |
| if: always() | |
| run: ls -ld $GITHUB_WORKSPACE |