diff --git a/.github/actions/audit-github-commit/action.yml b/.github/actions/audit-github-commit/action.yml new file mode 100644 index 00000000..10a87033 --- /dev/null +++ b/.github/actions/audit-github-commit/action.yml @@ -0,0 +1,41 @@ +name: "Audit GitHub commit" +description: "Get basic metadata associated with a commit" + +inputs: + sha: + description: "The commit to audit" + required: true +outputs: + tag: + description: "The associated release tag, if the release exists" + value: ${{ steps.commit.outputs.tag_name }} + release-id: + description: "The associated release id, if the release exists" + value: ${{ steps.commit.outputs.id }} + pre-release: + description: "If the associated release exists, is it a pre-release?" + value: ${{ steps.commit.outputs.prerelease }} + commitish: + description: "The associated commitish, if the release exists" + value: ${{ steps.commit.outputs.target_commitish }} + +runs: + using: composite + steps: + - name: "Check if a release exists for this commit" + id: commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: cardinalby/git-get-release-action@1.2.4 + with: + commitSha: ${{ inputs.sha }} + doNotFailIfNotFound: true # returns blank outputs when not found instead of error + searchLimit: 15 # Since we only care about recent releases, speed up the process + + - name: "[DEBUG] Print Release Details" + shell: bash + run: | + echo tag : ${{ steps.check_release_commit.outputs.tag_name }} + echo release-id : ${{ steps.check_release_commit.outputs.id }} + echo pre-release : ${{ steps.check_release_commit.outputs.prerelease }} + echo commitish : ${{ steps.check_release_commit.outputs.target_commitish }} diff --git a/.github/actions/audit-github-tag/action.yml b/.github/actions/audit-github-tag/action.yml new file mode 100644 index 00000000..6b8c747f --- /dev/null +++ b/.github/actions/audit-github-tag/action.yml @@ -0,0 +1,74 @@ +# When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a +# message of "release not found". In our case, it's not an actual error, just a +# confirmation that the release does not already exist, so we can create it. +# The `|| true` makes it so the step does not exit with a non-zero exit code. +# Also check if the release already exists is draft state. If it's a draft, +# and we are not testing, then we can publish that draft as-is. If it's in draft, +# and we are testing, skip the release. +name: "Audit GitHub tag" +description: "Get basic metadata associated with a release tag" + +inputs: + tag: + description: "The tag to audit (i.e. v1.0.0b1)" + required: true + repo-url: + description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + required: true +outputs: + exists: + description: "Does the tag exist?" + value: ${{ steps.tag.outputs.exists }} + sha: + description: "The commit associated with the tag if the tag exists" + value: ${{ steps.commit.outputs.sha }} + is-draft: + description: "If the tag exists, is it a draft?" + value: ${{ steps.draft.outputs.is-draft }} + +runs: + using: composite + steps: + - name: "Check if tag exists" + id: tag + shell: bash + run: | + output=$((gh release view ${{ inputs.tag }} --json isDraft,targetCommitish --repo ${{ inputs.repo-url }}) 2>&1) || true + echo "results=$output" >> $GITHUB_OUTPUT + if [[ "$output" == "release not found" ]] + then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Get commit associate with the tag" + id: commit + if: ${{ fromJSON(steps.tag.outputs.exists) == true }} + shell: bash + run: | + sha=$(jq -r '.targetCommitish' <<< "$output") + echo "sha=$sha" >> $GITHUB_OUTPUT + + - name: "Check if tag is a draft release" + id: draft + if: ${{ fromJSON(steps.tag.outputs.exists) == true }} + shell: bash + run: | + is-draft=$(jq -r '.isDraft' <<< "${{ steps.tag.outputs.results }}") + if [[ $is-draft == true ]] + then + echo "is-draft=true" >> $GITHUB_OUTPUT + else + echo "is-draft=false" >> $GITHUB_OUTPUT + fi + + - name: "[DEBUG] Tag metadata" + shell: bash + run: | + echo tag : ${{ inputs.tag }} + echo exists: : ${{ steps.release.outputs.exists }} + echo sha : ${{ steps.commit.outputs.sha }} + echo is-draft : ${{ steps.draft.outputs.is-draft }} diff --git a/.github/actions/build-archive-name/action.yml b/.github/actions/build-archive-name/action.yml new file mode 100644 index 00000000..d51f4fcb --- /dev/null +++ b/.github/actions/build-archive-name/action.yml @@ -0,0 +1,28 @@ +name: "Build" +description: "Provide a standard archive name for build artifacts" + +inputs: + package: + description: "Choose what to publish" + required: true + version: + description: "The release version number (i.e. 1.0.0b1)" + required: true + deploy-to: + description: "Choose where to publish" + default: prod +outputs: + archive-name: + description: "A unique archive name" + value: ${{ steps.archive.outputs.archive-name }} + +runs: + using: composite + + steps: + - name: "Archive name" + id: archive + shell: bash + run: | + archive_name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "archive-name=$archive_name" >> $GITHUB_OUTPUT diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml deleted file mode 100644 index fe9825d4..00000000 --- a/.github/actions/build-hatch/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build - `hatch` -description: Build artifacts using the `hatch` build backend - -inputs: - build-command: - description: The command to build distributable artifacts - default: "hatch build" - check-command: - description: The command to check built artifacts - default: "hatch run build:check-all" - working-dir: - description: Where to run commands from, supports namespace packaging - default: "./" - archive-name: - description: Where to upload the artifacts - required: true - -runs: - using: composite - steps: - - - name: Build artifacts - run: ${{ inputs.build-command }} - shell: bash - working-directory: ${{ inputs.working-dir }} - - - name: Check artifacts - run: ${{ inputs.check-command }} - shell: bash - working-directory: ${{ inputs.working-dir }} - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.archive-name }} - path: ${{ inputs.working-dir }}dist/ - retention-days: 3 diff --git a/.github/actions/publish-github-draft/action.yml b/.github/actions/publish-github-draft/action.yml new file mode 100644 index 00000000..0a1f77cf --- /dev/null +++ b/.github/actions/publish-github-draft/action.yml @@ -0,0 +1,72 @@ +name: "Publish to GitHub as draft release" +description: "Publish artifacts from an archive to GitHub as a draft release" + +inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" + tag: + description: "The release tag to publish under" + required: true + repo-url: + description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + required: true + sha: + description: "Commit SHA being released" + required: true + changelog-path: + description: "Path to the release notes" + required: true + +runs: + using: composite + steps: + + - name: "Download artifacts" + if: ${{ !(inputs.archive-name == "" }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.archive-name }} + path: dist/ + + - name: "[INFO] Downloaded artifacts" + if: ${{ !(inputs.archive-name == "" }} + shell: bash + run: | + title="Downloaded artifacts" + message="Downloaded artifacts from ${{ inputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Found artifacts" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: echo $(ls ./dist) + + - name: "Set: pre-release" + id: pre-release + shell: bash + run: | + if ${{ contains(inputs.tag, 'rc') || contains(inputs.tag, 'b') || contains(inputs.tag, 'a') }} + then + echo "pre-release=--prerelease" >> $GITHUB_OUTPUT + fi + + - name: "Publish artifacts to GitHub as draft release" + shell: bash + run: | + gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE --draft --repo ${{ inputs.repo-url }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ inputs.tag }} + TITLE: ${{ github.event.repository.name }} ${{ inputs.tag }} + RELEASE_NOTES: ${{ inputs.changelog-path }} + COMMIT: ${{ inputs.sha }} + PRERELEASE: ${{ steps.pre-release.outputs.pre-release }} + DRAFT: ${{ inputs.draft-flag }} + + - name: "[INFO] Published artifacts to GitHub as draft release" + shell: bash + run: | + title="Published artifacts to GitHub as draft release" + message="artifacts: $(ls ./dist) tag: ${{ inputs.tag }} pre-release: ${{ steps.prerelease.outputs.prerelease }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-github-full/action.yml b/.github/actions/publish-github-full/action.yml new file mode 100644 index 00000000..4dc207d8 --- /dev/null +++ b/.github/actions/publish-github-full/action.yml @@ -0,0 +1,26 @@ +name: "Publish GitHub draft release as full release" +description: "Publish an existing draft release as a non-draft release" + +inputs: + tag: + description: "The tag to publish (i.e. v1.0.0b1)" + required: true + repo-url: + description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + required: true + +runs: + using: composite + steps: + - name: "Publish draft release" + shell: bash + run: gh release edit ${{ inputs.tag }} --draft=false --repo ${{ inputs.repo-url }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "[INFO] Released draft as non-draft release on GitHub" + shell: bash + run: | + title="Released draft as non-draft release on GitHub" + message="tag: ${{ inputs.tag }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index deffc6e3..44019426 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -1,29 +1,51 @@ -name: Publish - PyPI -description: Publish artifacts saved during build step to PyPI +name: "Publish to PyPI" +description: "Publish artifacts from an archive to PyPI" inputs: archive-name: - description: Where to download the artifacts from - required: true + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" repository-url: - description: The PyPI index to publish to, test or prod + description: "The url for the PyPI index" required: true + working-dir: + description: "Where to run commands from, primarily supports namespace packaging" + default: "./" runs: using: composite steps: - - name: Download artifacts - uses: actions/download-artifact@v3 + - name: "Download artifacts" + if: ${{ !(inputs.archive-name == "" }} + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive-name }} path: dist/ - - name: "[DEBUG]" - run : ls -R + - name: "[INFO] Downloaded artifacts" + if: ${{ !(inputs.archive-name == "" }} + shell: bash + run: | + title="Downloaded artifacts" + message="Downloaded artifacts from ${{ inputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Found artifacts" shell: bash + working-directory: ${{ inputs.working-dir }} + run: echo $(ls ./dist) - - name: Publish artifacts to PyPI + - name: "Publish artifacts to PyPI" uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: ${{ inputs.repository-url }} + packages-dir: ${{ inputs.working-dir }}dist/ + + - name: "[INFO] Published artifacts to PyPI" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: | + title="Published artifacts to PyPI" + message=$(ls ./dist) + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml deleted file mode 100644 index d863d659..00000000 --- a/.github/actions/publish-results/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish results - -inputs: - file-name: - description: File type for file name stub (e.g. "unit-tests") - required: true - python-version: - description: Python version for the file name stub (e.g. "3.8") - required: true - source-file: - description: File to be uploaded - required: true - -runs: - using: composite - steps: - - name: Get timestamp - id: timestamp - run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - shell: bash - - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.file-name }}_python-${{ inputs.python-version }}_${{ steps.timestamp.outputs.ts }}.csv - path: ${{ inputs.source-file }} diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-results/action.yml new file mode 100644 index 00000000..0ec18bb4 --- /dev/null +++ b/.github/actions/publish-test-results/action.yml @@ -0,0 +1,48 @@ +name: "Publish test results" +description: "Upload test artifacts to a unique archive name" + +inputs: + archive-type: + description: "File type for archive name stub (e.g. unit-tests)" + required: true + python-version: + description: "Python version for the archive name stub (e.g. 3.8)" + required: true + os: + description: "OS for the archive name stub (e.g. ubuntu-latest)" + required: true + source-file: + description: "File/directory to be uploaded" + required: true +outputs: + archive-name: + description: "Name of the archive containing the test artifacts" + value: ${{ steps.archive.outputs.archive-name }} + +runs: + using: composite + steps: + - name: "Get timestamp for archive name" + id: timestamp + shell: bash + run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts + + - name: "Build archive name" + id: archive + shell: bash + run: | + archive_name = ${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_${{ steps.timestamp.outputs.ts }} + echo "archive-name=archive_name" >> $GITHUB_OUTPUT + + - name: "Upload test results" + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.archive.outputs.archive-name }} + path: ${{ inputs.source-file }} + + - name: "[INFO] Uploaded test artifacts" + shell: bash + run: | + title="Uploaded test artifacts" + message="Uploaded ${{ inputs.source-file }} to ${{ steps.archive.outputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml index 7b7780ef..8c45ec98 100644 --- a/.github/actions/setup-hatch/action.yml +++ b/.github/actions/setup-hatch/action.yml @@ -1,22 +1,22 @@ -name: Setup - `hatch` -description: Setup a python environment with `hatch` installed +name: "Setup `hatch`" +description: "Setup a python environment with `hatch` installed" inputs: setup-command: - description: The command to setup development dependencies + description: "The command to setup development dependencies" default: "python -m pip install hatch" python-version: - description: The version of python to install + description: "The version of python to install" default: "3.11" runs: using: composite steps: - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 + - name: "Set up python ${{ inputs.python-version }}" + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - - name: Install dev dependencies - run: ${{ inputs.setup-command }} + - name: "Install build dependencies" shell: bash + run: ${{ inputs.setup-command }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33d94ff4..30c6e23f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,85 +1,87 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. - name: "Build" +run-name: "Build ${{ inputs.package }}==${{ inputs.version}} for deployment to ${{ deploy-to }}" on: push: branches: - "main" - pull_request: merge_group: types: [checks_requested] workflow_dispatch: + inputs: + package: + description: "Choose what to publish" + type: choice + options: + - dbt-adapters + - dbt-tests-adapter + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: choice + options: + - prod + - test workflow_call: inputs: - changelog_path: - description: "Path to changelog file" + package: + description: "Choose what to publish (dbt-adapters, dbt-tests-adapter)" + type: string + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string required: true + deploy-to: + description: "Choose where to publish" type: string + default: prod + outputs: + archive-name: ${{ jobs.build.outputs.archive-name }} permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true defaults: run: shell: bash +env: + NOTIFICATION_PREFIX: "[Build]" + jobs: build: - name: Build, Test and publish to PyPi + name: "Build, test and publish to PyPi" runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + outputs: + archive-name: ${{ steps.archive.outputs.archive-name }} + steps: - name: "Check out repository" uses: actions/checkout@v4 - - name: Setup `hatch` + - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" + - name: "Archive name" + id: archive + uses: ./.github/actions/build-archive-name - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` + - name: "Build `dbt-adapters`" if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch + uses: ./.github/actions/build + with: + archive-name: ${{ steps.archive.outputs.archive-name }} - - name: Build `dbt-tests-adapter` + - name: "Build `dbt-tests-adapter`" if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch + uses: ./.github/actions/build with: + archive-name: ${{ steps.archive.outputs.archive-name }} working-dir: "./dbt-tests-adapter/" - - # this step is only needed for the release process - - name: "Upload Build Artifact" - if: ${{ github.event_name == 'workflow_call' }} - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.version.outputs.version_number }} - path: | - ${{ inputs.changelog_path }} - ./dist/ - retention-days: 3 diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml index d778f565..b83bd11e 100644 --- a/.github/workflows/changelog-existence.yml +++ b/.github/workflows/changelog-existence.yml @@ -14,13 +14,12 @@ # and when new code is pushed to the branch. The action will get # skipped if the 'Skip Changelog' label is present is any of the labels. -name: Check Changelog Entry +name: "Check for changelog entry" on: - pull_request_target: + pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - workflow_dispatch: defaults: @@ -37,4 +36,4 @@ jobs: with: changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' skip_label: 'Skip Changelog' - secrets: inherit + secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 4f5b392e..1c712eec 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -1,4 +1,4 @@ -name: Code Quality +name: "Code quality" on: push: @@ -21,20 +21,20 @@ concurrency: jobs: lint: - name: Code Quality + name: "Code quality" runs-on: ubuntu-latest steps: - - name: Check out repository + - name: "Check out repository" uses: actions/checkout@v4 with: persist-credentials: false - - name: Setup `hatch` + - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - - name: Run linters + - name: "Run linters" run: hatch run lint:all - - name: Run typechecks + - name: "Run typechecks" run: hatch run typecheck:all diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml deleted file mode 100644 index edd25172..00000000 --- a/.github/workflows/github-release.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# Create a new release on GitHub and include any artifacts in the `/dist` directory of the GitHub artifacts store. -# -# Inputs: -# sha: The commit to attach to this release -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# changelog_path: Path to the changelog file for release notes -# test_run: Test run (Publish release as draft) -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call after a successful build. Build artifacts should be ready to release and live in a dist/ directory. -# -# This workflow expects the artifacts to already be built and living in the artifact store of the workflow. -# -# Validation Checks -# -# 1. If no release already exists for this commit and version, create the tag and release it to GitHub. -# 2. If a release already exists for this commit, skip creating the release but finish with a success. -# 3. If a release exists for this commit under a different tag, fail. -# 4. If the commit is already associated with a different release, fail. - -name: GitHub Release - -on: - workflow_call: - inputs: - sha: - description: The commit to attach to this release - required: true - type: string - version_number: - description: The release version number (i.e. 1.0.0b1) - required: true - type: string - changelog_path: - description: Path to the changelog file for release notes - required: true - type: string - test_run: - description: Test run (Publish release as draft) - required: true - type: boolean - archive_name: - description: artifact name to download - required: true - type: string - outputs: - tag: - description: The path to the changelog for this version - value: ${{ jobs.check-release-exists.outputs.tag }} - -permissions: - contents: write - -env: - REPO_LINK: ${{ github.server_url }}/${{ github.repository }} - NOTIFICATION_PREFIX: "[GitHub Release]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The release version number: ${{ inputs.version_number }} - echo Expected Changelog path: ${{ inputs.changelog_path }} - echo Test run: ${{ inputs.test_run }} - echo Repo link: ${{ env.REPO_LINK }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - - check-release-exists: - runs-on: ubuntu-latest - outputs: - exists: ${{ steps.release_check.outputs.exists }} - draft_exists: ${{ steps.release_check.outputs.draft_exists }} - tag: ${{ steps.set_tag.outputs.tag }} - - steps: - - name: "Generate Release Tag" - id: set_tag - run: echo "tag=v${{ inputs.version_number }}" >> $GITHUB_OUTPUT - - # When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a - # message of "release not found". In our case, it's not an actual error, just a - # confirmation that the release does not already exists so we can go ahead and create it. - # The `|| true` makes it so the step does not exit with a non-zero exit code - # Also check if the release already exists is draft state. If it does, and we are not - # testing then we can publish that draft as is. If it's in draft and we are testing, skip the - # release. - - name: "Check If Release Exists For Tag ${{ steps.set_tag.outputs.tag }}" - id: release_check - run: | - output=$((gh release view ${{ steps.set_tag.outputs.tag }} --json isDraft,targetCommitish --repo ${{ env.REPO_LINK }}) 2>&1) || true - if [[ "$output" == "release not found" ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} does not exist." - message="Check passed." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - commit=$(jq -r '.targetCommitish' <<< "$output") - if [[ $commit != ${{ inputs.sha }} ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists for commit $commit!" - message="Cannot create a new release for commit ${{ inputs.sha }}. Exiting." - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - isDraft=$(jq -r '.isDraft' <<< "$output") - if [[ $isDraft == true ]] && [[ ${{ inputs.test_run }} == false ]] - then - title="Release tag ${{ steps.set_tag.outputs.tag }} already associated with the draft release." - message="Release workflow will publish the associated release." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=true" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists." - message="Skip GitHub Release Publishing." - echo "exists=true" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ env.REPO_LINK }} - - - name: "[DEBUG] Log Job Outputs" - run: | - echo exists: ${{ steps.release_check.outputs.exists }} - echo draft_exists: ${{ steps.release_check.outputs.draft_exists }} - echo tag: ${{ steps.set_tag.outputs.tag }} - - skip-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'true' - - steps: - - name: "Tag Exists, Skip GitHub Release Job" - run: | - echo title="A tag already exists for ${{ needs.check-release-exists.outputs.tag }} and commit." - echo message="Skipping GitHub release." - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - audit-release-different-commit: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'false' - - steps: - - name: "Check If Release Already Exists For Commit" - uses: cardinalby/git-get-release-action@1.2.4 - id: check_release_commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process - - - name: "[DEBUG] Print Release Details" - run: | - echo steps.check_release_commit.outputs.id: ${{ steps.check_release_commit.outputs.id }} - echo steps.check_release_commit.outputs.tag_name: ${{ steps.check_release_commit.outputs.tag_name }} - echo steps.check_release_commit.outputs.target_commitish: ${{ steps.check_release_commit.outputs.target_commitish }} - echo steps.check_release_commit.outputs.prerelease: ${{ steps.check_release_commit.outputs.prerelease }} - - # Since we already know a release for this tag does not exist, if we find anything it's for the wrong tag, exit - - name: "Check If The Tag Matches The Version Number" - if: steps.check_release_commit.outputs.id != '' - run: | - title="Tag ${{ steps.check_release_commit.outputs.tag_name }} already exists for this commit!" - message="Cannot create a new tag for ${{ needs.check-release-exists.outputs.tag }} for the same commit" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - - publish-draft-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: >- - needs.check-release-exists.outputs.draft_exists == 'true' && - inputs.test_run == false - - steps: - - name: "Publish Draft Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release edit $TAG --draft=false --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - - create-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: needs.check-release-exists.outputs.draft_exists == 'false' - - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.archive_name }} - path: . - - - name: "[DEBUG] Display Structure Of Expected Files" - run: | - ls -R .changes - ls -l dist - - - name: "Set Release Type" - id: release_type - run: | - if ${{ contains(inputs.version_number, 'rc') || contains(inputs.version_number, 'b') }} - then - echo Release will be set as pre-release - echo "prerelease=--prerelease" >> $GITHUB_OUTPUT - else - echo This is not a prerelease - fi - - - name: "Set As Draft Release" - id: draft - run: | - if [[ ${{ inputs.test_run }} == true ]] - then - echo Release will be published as draft - echo "draft=--draft" >> $GITHUB_OUTPUT - else - echo This is not a draft release - fi - - - name: "GitHub Release Workflow Annotation" - run: | - title="Release ${{ needs.check-release-exists.outputs.tag }}" - message="Configuration: ${{ steps.release_type.outputs.prerelease }} ${{ steps.draft.outputs.draft }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "Create New GitHub Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - TITLE: ${{ github.event.repository.name }} ${{ needs.check-release-exists.outputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog_path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.release_type.outputs.prerelease }} - DRAFT: ${{ steps.draft.outputs.draft }} \ No newline at end of file diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml new file mode 100644 index 00000000..398ae450 --- /dev/null +++ b/.github/workflows/publish-github.yml @@ -0,0 +1,159 @@ +# **what?** +# 1. If a release exists for this commit under a different tag, fail. +# 2. If the commit is already associated with a different release, fail. +# 3. If no release/tag exists for this commit and version, create the tag and release it to GitHub as a draft. +# 4. If a release already exists for this commit, skip creating the release as a draft. +# 5. If this is deployed to prod, mark the draft release (created in this run or previously) as a full release. +# +# **when?** +# 1. Call after a successful build. Build artifacts should be ready to release.defaults: +# They need to either live in a dist/ directory locally, or an archive name needs to be provided. +# 2. Call to publish a draft release to become a full release. No archive or artifacts are needed in this case. +name: "Publish to GitHub" +run-name: "Publish `${{ version }}` to GitHub in ${{ deploy-to }}" + +on: + workflow_dispatch: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "Commit SHA being released" + type: string + required: true + changelog-path: + description: "Path to the release notes" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: choice + options: + - "prod" + - "test" + workflow_call: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "Commit SHA being released" + type: string + required: true + changelog-path: + description: "Path to the release notes" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: string + default: "prod" + +permissions: + contents: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + cancel-in-progress: true + +env: + REPO_LINK: ${{ github.server_url }}/${{ github.repository }} + NOTIFICATION_PREFIX: "[GitHub Release]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo Expected Changelog path: ${{ inputs.changelog_path }} + echo Test run: ${{ inputs.test_run }} + echo Repo link: ${{ env.REPO_LINK }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + publish: + runs-on: ubuntu-latest + outputs: + exists: ${{ steps.release_check.outputs.exists }} + draft_exists: ${{ steps.release_check.outputs.draft_exists }} + tag: ${{ steps.set_tag.outputs.tag }} + + steps: + - name: "Build tag" + id: tag + shell: bash + run: echo "tag=v${{ inputs.version }}" >> $GITHUB_OUTPUT + + - name: "Get tag metadata" + id: tag-metadata + uses: ./.github/actions/audit-github-tag + with: + tag: ${{ steps.tag.output.tag }} + repo-url: ${{ env.REPO_LINK }} + + - name: "[INFO] Release already exists" + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} + id: release_check + run: | + title="Tag already exists, skip publishing artifacts" + message="Tag ${{ steps.tag.outputs.tag }} already exists and is associated with the commit ${{ steps.tag-metadata.outputs.sha }}." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[ERROR] Exit if the tag exists an is associated with a different commit than the one provided" + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} != ${{ inputs.sha }} + shell: bash + run: | + title="Tag ${{ inputs.tag }} already exists with a different commit!" + message="Existing commit: {{ steps.tag-metadata.outputs.sha }} New commit: ${{ inputs.sha }}. Exiting." + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + + - name: "Get commit metadata" + id: commit-metadata + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} + uses: ./.github/actions/audit-github-commit + with: + tag: ${{ inputs.sha }} + + - name: "[ERROR] Exit if the commit is associated with a different tag than the version that was provided" + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false && steps.commit-metadata.outputs.tag != '' }} + run: | + title="Commit ${{ inputs.sha }} is already associated with a different tag!" + message="Existing tag: ${{ steps.commit-metadata.outputs.tag }} New tag: ${{ steps.tag.outputs.tag }}" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + + - name: "Publish to GitHub as draft release" + if: fromJSON(steps.tag-metadata.outputs.exists) == false + uses: ./.github/actions/publish-github-draft + with: + archive-name: ${{ inputs.archive-name }} + tag: ${{ needs.check-release-exists.outputs.tag }} + repo-url: ${{ env.REPO_LINK }} + sha: ${{ inputs.sha }} + changelog-path: ${{ inputs.changelog_path }} + + - name: "Publish draft release as full release" + if: >- + inputs.deploy-to == "prod" && ( + fromJSON(steps.tag-metadata.outputs.exists) == false || + fromJSON(steps.tag-metadata.outputs.is-draft) == true + ) + uses: ./.github/actions/publish-github-full + with: + tag: ${{ steps.tag.output.tag }} + repo-url: ${{ env.REPO_LINK }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 00000000..509a699c --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,75 @@ +# **what?** +# Releases artifacts to PyPI. +# +# **when?** +# Call after a successful build. Build artifacts should be ready to release.defaults: +# They need to either live in a dist/ directory locally, or an archive name needs to be provided. +name: "Publish to PyPI" +run-name: "Publish `${{ version }}` to PyPI in ${{ deploy-to }}" + +on: + workflow_dispatch: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: choice + options: + - "prod" + - "test" + workflow_call: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: string + default: "prod" + +permissions: + contents: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + cancel-in-progress: true + +jobs: + log-inputs: + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo archive-name : ${{ inputs.archive-name }} + echo version : ${{ inputs.version }} + echo deploy-to : ${{ inputs.deploy-to }} + echo PYPI_PROJECT_URL : ${{ vars.PYPI_PROJECT_URL }} + echo PYPI_REPOSITORY_URL : ${{ vars.PYPI_REPOSITORY_URL }} + + pypi-release: + name: "Publish to PyPI" + runs-on: ubuntu-latest + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + steps: + - name: "Check out repository" + uses: actions/checkout@v4 + + - name: "Publish to PyPI" + uses: .github/actions/publish-pypi + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1135adb8..d140a9ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,70 +1,64 @@ -name: Release -run-name: Release ${{ inputs.package }}==${{ inputs.version_number }} to ${{ inputs.deploy-to }} +name: "Release" +run-name: "Release `${{ inputs.package }}==${{ inputs.version }}` to ${{ inputs.deploy-to }}" on: workflow_dispatch: inputs: package: + description: "Choose what to release" type: choice - description: Choose what to publish options: - - dbt-adapters - - dbt-tests-adapter - version_number: + - "dbt-adapters" + - "dbt-tests-adapter" + target-branch: + description: "Choose what branch to release from" + type: string + default: "main" + version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true deploy-to: + description: "Choose where to publish" type: choice - description: Choose where to publish options: - - prod - - test - default: prod - nightly_release: - description: "Nightly release to dev environment" + - "prod" + - "test" + nightly-release: + description: "Is this the nightly release to the dev environment?" type: boolean default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - workflow_call: inputs: package: + description: "Choose what to release" type: string - description: Choose what to publish - required: true - version_number: + default: "dbt-adapters" + target-branch: + description: "Choose what branch to release from" + type: string + default: "main" + version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true deploy-to: + description: "Choose where to publish" type: string - default: prod - required: false - nightly_release: - description: "Nightly release to dev environment" + default: "prod" + nightly-release: + description: "Is this the nightly release to the dev environment?" type: boolean default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main -# this is the permission that allows creating a new release +# this is the permission that allows creating a new release to both GitHub and PyPI permissions: contents: write id-token: write -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +# will cancel previous workflows triggered by the same event and for the same package and environment concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.package }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true defaults: @@ -128,53 +122,33 @@ jobs: echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build-and-test: - name: "Build and Test" + build-artifacts: + name: "Build ${{ inputs.package }} ${{ inputs.version }}" needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - - name: "Build ${{ inputs.package }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main - with: - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + uses: dbt-labs/dbt-adapters/.github/workflows/build.yml@update-workflows + with: + package: ${{ inputs.package }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} - github-release: - name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + # TODO: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + publish-github: + name: "Publish to GitHub" if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [release-inputs, build-and-test, bump-version-generate-changelog] - uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main + needs: [build-artifacts, bump-version-generate-changelog] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows with: + archive-name: ${{ needs.build-artifacts.outputs.archive-name }} + version: ${{ inputs.version }} sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.deploy-to == 'test' && true || false }} - archive_name: ${{ needs.release-inputs.outputs.archive-name }} + changelog-path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + deploy-to: ${{ inputs.deploy-to }} - pypi-release: + publish-0pypi: name: "Publish to PyPI" - runs-on: ubuntu-latest - needs: [release-inputs, build-and-test] - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }} - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: "Publish to PyPI" - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main - with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + needs: [build-artifacts] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + with: + archive-name: ${{ needs.build-artifacts.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} diff --git a/.github/workflows/resubmit-for-triage.yml b/.github/workflows/resubmit-for-triage.yml index 385ef820..3020509c 100644 --- a/.github/workflows/resubmit-for-triage.yml +++ b/.github/workflows/resubmit-for-triage.yml @@ -10,7 +10,7 @@ # **when?** # This will run when a comment is added to an issue and that issue has an `awaiting_response` label. -name: Resubmit for Triage +name: "Resubmit for triage" on: issue_comment @@ -22,7 +22,7 @@ permissions: issues: write jobs: - triage_label: + triage-label: if: contains(github.event.issue.labels.*.name, 'awaiting_response') uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main with: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 26ff4aaa..5022a0c6 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,4 +1,4 @@ -name: Unit Tests +name: "Unit tests" on: push: @@ -17,33 +17,35 @@ concurrency: jobs: unit: - name: Unit Tests - runs-on: ubuntu-latest + name: "Unit tests" + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-latest] steps: - - name: Check out repository + - name: "Check out repository" uses: actions/checkout@v4 with: persist-credentials: false - - name: Setup `hatch` + - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch with: python-version: ${{ matrix.python-version }} - - name: Run unit tests + - name: "Run unit tests" run: hatch run unit-tests:all shell: bash - - name: Publish results - uses: ./.github/actions/publish-results + - name: "Publish results" + uses: ./.github/actions/publish-test-results if: always() with: - source-file: "results.csv" - file-name: "unit_results" + archive-type: "unit_results" python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + source-file: "results.csv"