From 8cc87a788eb65aea12d2d8556c3120bd458200b4 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 12:41:19 -0800 Subject: [PATCH 01/12] feat: add common dynamic versioning workflow --- .github/workflows/release_workflow.yml | 61 ++++++++++++++++++++++++++ README.md | 15 +++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/workflows/release_workflow.yml diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml new file mode 100644 index 0000000..12567a8 --- /dev/null +++ b/.github/workflows/release_workflow.yml @@ -0,0 +1,61 @@ +name: Publish GitHub Release to PyPi + +# This workflow executes upon the publishing of a release in GitHub. This does not run on draft releases. +# +# Assumptions: +# +# 1. The repo is a python project which can be deployed using the Python version below noted in `python-version: __`. +# 2. The `pyproject.toml` file should declare `version = 0.0.0` in deference of dynamic versioning. +# 3. The release title should be in the semvar format (`vX.Y.Z`) +# 4. The repo secrets should declare (or have access to) the following env vars / secrets: +# 1. TEST_PYPI_TOKEN - A token to use for test publish. +# 2. PYPI_TOKEN - A token to use for actual publish. +# +# Other notes: +# +# - Unless you block the hotkey, you browser will likely interpret "Ctrl+Enter" as "Publish" and not "Save Draft", +# even if you are editing an existing draft. +# - The text copy of Release Notes can freely be edited after publish, with no impact on the publish process and without +# risk of accidentally republishing. +# - PyPi will only accept each semvar increment once, but failures to publish can be freely retried, and you can yank +# and release a patch on top if needed. + +on: + release: + types: [published] + +jobs: + build_deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + - name: Build package + run: | + poetry self add "poetry-dynamic-versioning[plugin]" + poetry config repositories.testpypi https://test.pypi.org/legacy/ + poetry dynamic-versioning --no-cache + poetry build + - name: Upload wheel to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: dist/*.whl + tag: ${{ github.ref }} + overwrite: true + file_glob: true + - name: Deploy to PyPI + run: | + poetry publish -r testpypi -u "__token__" -p "${{ secrets.TEST_PYPI_TOKEN }}" + poetry publish -u "__token__" -p "${{ secrets.PYPI_TOKEN }}" \ No newline at end of file diff --git a/README.md b/README.md index 337cff0..d7e71da 100644 --- a/README.md +++ b/README.md @@ -148,3 +148,18 @@ Check out all the [published connectors](https://hub.meltano.com/singer/maintain | [target-yaml](https://github.com/MeltanoLabs/target-yaml) | Prerelease (Beta) | 🔭 Looking For Maintainers! 🔭 | [Link](https://hub.meltano.com/targets/yaml--meltanolabs) | | [target-snowflake](https://github.com/MeltanoLabs/target-snowflake) | Prerelease (Beta) | [Meltano](https://github.com/meltano) - Maintainer | - | | [target-jsonl-blob](https://github.com/MeltanoLabs/target-jsonl-blob) | Prerelease (Beta) | [Edgar Ramírez](https://github.com/edgarrmondragon) - Maintainer | - | + +## Common CI Workflows + +MeltanoLabs taps and targets can share workflows via the GitHub [Required Workflows](https://github.blog/2023-01-10-introducing-required-workflows-and-configuration-variables-to-github-actions/#:~:text=Required%20workflows%20allows%20DevOps%20teams,impossible%20task%20in%20large%20organizations.) feature. + +Repositories are "opted in" to shared workflows via the [Add Required Workflows](https://github.com/organizations/MeltanoLabs/settings/actions/required_workflows/new) GitHub web UI. + +When eligible for this behavior, and when the repo is opted-in by a MeltanoLabs administrator, common CI workflows may include (but are not necessarily limited to): + +1. Automated CI testing on currently-supported Python versions. +1. Automated verification of Semantic PR logic. +1. Release management using dynamic versioning. +1. Automated Release Notes drafting using Semantic PRs. + +For a full list of common workflows, see the `.github/workflows` directory within this repo. Each workflow should be self-documenting, including assumptions and behavior notes. From 93ab7aee4e7ed1b91103c2d0d596b6592fc4cef6 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 13:30:06 -0800 Subject: [PATCH 02/12] rename file --- .../{release_workflow.yml => common_release_workflow.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{release_workflow.yml => common_release_workflow.yml} (97%) diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/common_release_workflow.yml similarity index 97% rename from .github/workflows/release_workflow.yml rename to .github/workflows/common_release_workflow.yml index 12567a8..d39d3e8 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/common_release_workflow.yml @@ -1,4 +1,4 @@ -name: Publish GitHub Release to PyPi +name: Publish GitHub Release to PyPi (MeltanoLabs) # This workflow executes upon the publishing of a release in GitHub. This does not run on draft releases. # From e777ecd2fb60b60b35d1322ec9f2edd34b06f434 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 13:45:40 -0800 Subject: [PATCH 03/12] feat: add default semantic PR settings --- .github/semantic.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/semantic.yml diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 0000000..dab49a0 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,24 @@ +## This is a default config file for the Semantic PRs GitHub App. + +# Config ref: https://github.com/Ezard/semantic-prs + +# Validate the PR title, and ignore all commit messages +titleOnly: true + +# Provides a custom URL for the "Details" link, which appears next to the success/failure message from the app: +targetUrl: https://sdk.meltano.com/en/latest/CONTRIBUTING.html#semantic-pull-requests + +# The values allowed for the "type" part of the PR title/commit message. +# e.g. for a PR title/commit message of "feat: add some stuff", the type would be "feat" +types: + - ci + - chore + - build + - docs + - feat + - fix + - perf + - refactor + - revert + - style + - test From 554d832fd1b2aa0884a61f0e60bb461e4a3b473f Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 13:48:43 -0800 Subject: [PATCH 04/12] chore: add note --- .github/semantic.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/semantic.yml b/.github/semantic.yml index dab49a0..e1e9a8a 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -1,4 +1,8 @@ ## This is a default config file for the Semantic PRs GitHub App. +## +## Note: until the related issue is closed, this file still needs to be copied to each repo. +## Otherwise, the default configuration from the Semantic PRs GitHub App will be used. +## See: https://github.com/Ezard/semantic-prs/issues/227 # Config ref: https://github.com/Ezard/semantic-prs From 0e3140486b70635c9aeb1301137e761d09c52efe Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 13:53:29 -0800 Subject: [PATCH 05/12] feat: add lint and pytest --- .github/workflows/common_pytest.yml | 39 ++++++++++++++++++++ .github/workflows/common_python_tox_lint.yml | 37 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/workflows/common_pytest.yml create mode 100644 .github/workflows/common_python_tox_lint.yml diff --git a/.github/workflows/common_pytest.yml b/.github/workflows/common_pytest.yml new file mode 100644 index 0000000..22497d6 --- /dev/null +++ b/.github/workflows/common_pytest.yml @@ -0,0 +1,39 @@ +# A common CI workflow that runs pytest on all actively supported Python versions. +# +# Assumptions: +# +# 1. The repo works with the below-specified version of Poetry. +# 2. The repo can be tested with pytest. +# 3. All Python versions named below are enabled for testing. + +name: Python Pytest Check (MeltanoLabs) + +on: [push] + +jobs: + pytest: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + strategy: + fail-fast: false + matrix: + python-version: [3.7, 3.8, 3.9, 3.10] + + steps: + - uses: actions/checkout@v3 + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.3.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + - name: Install dependencies + run: | + poetry install + - name: Test with pytest + run: | + poetry run pytest --capture=no \ No newline at end of file diff --git a/.github/workflows/common_python_tox_lint.yml b/.github/workflows/common_python_tox_lint.yml new file mode 100644 index 0000000..8e19574 --- /dev/null +++ b/.github/workflows/common_python_tox_lint.yml @@ -0,0 +1,37 @@ +# A common CI workflow to run linting as defined in tox.ini +# +# Assumptions: +# +# 1. The repo has a tox.ini file, which defines a 'lint' command. +# 2. The lint operation will be run on the below-named python version. + +name: Python Lint Checks from Tox (MeltanoLabs) + +on: [push] + +jobs: + linting: + + runs-on: ubuntu-latest + strategy: + matrix: + # Only lint using the primary version used for dev + python-version: [3.10] + + steps: + - uses: actions/checkout@v3 + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.3.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + - name: Install dependencies + run: | + poetry install + - name: Run lint command from tox.ini + run: | + poetry run tox -e lint From 82e86db7ce7069409d8a2ad599863f16bb624419 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 13:55:43 -0800 Subject: [PATCH 06/12] chore: 'common' subfolder --- .github/workflows/{ => common}/common_pytest.yml | 0 .github/workflows/{ => common}/common_python_tox_lint.yml | 0 .github/workflows/{ => common}/common_release_workflow.yml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ => common}/common_pytest.yml (100%) rename .github/workflows/{ => common}/common_python_tox_lint.yml (100%) rename .github/workflows/{ => common}/common_release_workflow.yml (100%) diff --git a/.github/workflows/common_pytest.yml b/.github/workflows/common/common_pytest.yml similarity index 100% rename from .github/workflows/common_pytest.yml rename to .github/workflows/common/common_pytest.yml diff --git a/.github/workflows/common_python_tox_lint.yml b/.github/workflows/common/common_python_tox_lint.yml similarity index 100% rename from .github/workflows/common_python_tox_lint.yml rename to .github/workflows/common/common_python_tox_lint.yml diff --git a/.github/workflows/common_release_workflow.yml b/.github/workflows/common/common_release_workflow.yml similarity index 100% rename from .github/workflows/common_release_workflow.yml rename to .github/workflows/common/common_release_workflow.yml From 4c9fdc492ece7146cd08dd7a6cb4f08afefc909e Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 20 Jan 2023 17:11:25 -0800 Subject: [PATCH 07/12] chore: migrate semantic.yml to https://github.com/MeltanoLabs/.github/pull/3 --- .github/semantic.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/semantic.yml diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index e1e9a8a..0000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,28 +0,0 @@ -## This is a default config file for the Semantic PRs GitHub App. -## -## Note: until the related issue is closed, this file still needs to be copied to each repo. -## Otherwise, the default configuration from the Semantic PRs GitHub App will be used. -## See: https://github.com/Ezard/semantic-prs/issues/227 - -# Config ref: https://github.com/Ezard/semantic-prs - -# Validate the PR title, and ignore all commit messages -titleOnly: true - -# Provides a custom URL for the "Details" link, which appears next to the success/failure message from the app: -targetUrl: https://sdk.meltano.com/en/latest/CONTRIBUTING.html#semantic-pull-requests - -# The values allowed for the "type" part of the PR title/commit message. -# e.g. for a PR title/commit message of "feat: add some stuff", the type would be "feat" -types: - - ci - - chore - - build - - docs - - feat - - fix - - perf - - refactor - - revert - - style - - test From e23f73c542bb0cfdaaea3cd416b146a8d61b8ecb Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 24 Jan 2023 11:27:29 -0800 Subject: [PATCH 08/12] Update .github/workflows/common/common_pytest.yml Co-authored-by: Will Da Silva --- .github/workflows/common/common_pytest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/common/common_pytest.yml b/.github/workflows/common/common_pytest.yml index 22497d6..9e09acc 100644 --- a/.github/workflows/common/common_pytest.yml +++ b/.github/workflows/common/common_pytest.yml @@ -35,5 +35,4 @@ jobs: run: | poetry install - name: Test with pytest - run: | - poetry run pytest --capture=no \ No newline at end of file + run: poetry run pytest --capture=no From 717f0431f7ccc36cbb25629b3f3e316388cdc164 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 24 Jan 2023 11:28:52 -0800 Subject: [PATCH 09/12] Update .github/workflows/common/common_release_workflow.yml Co-authored-by: Will Da Silva --- .github/workflows/common/common_release_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/common/common_release_workflow.yml b/.github/workflows/common/common_release_workflow.yml index d39d3e8..761a80b 100644 --- a/.github/workflows/common/common_release_workflow.yml +++ b/.github/workflows/common/common_release_workflow.yml @@ -58,4 +58,4 @@ jobs: - name: Deploy to PyPI run: | poetry publish -r testpypi -u "__token__" -p "${{ secrets.TEST_PYPI_TOKEN }}" - poetry publish -u "__token__" -p "${{ secrets.PYPI_TOKEN }}" \ No newline at end of file + poetry publish -u "__token__" -p "${{ secrets.PYPI_TOKEN }}" From 78a97c252381679adfc93ae0f6843dad62848c52 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 24 Jan 2023 11:35:04 -0800 Subject: [PATCH 10/12] Update .github/workflows/common/common_release_workflow.yml Co-authored-by: Will Da Silva --- .github/workflows/common/common_release_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/common/common_release_workflow.yml b/.github/workflows/common/common_release_workflow.yml index 761a80b..495a1d8 100644 --- a/.github/workflows/common/common_release_workflow.yml +++ b/.github/workflows/common/common_release_workflow.yml @@ -1,4 +1,4 @@ -name: Publish GitHub Release to PyPi (MeltanoLabs) +name: Publish GitHub Release to PyPI (MeltanoLabs) # This workflow executes upon the publishing of a release in GitHub. This does not run on draft releases. # From 57d45fc70e531383bf1d68655f797ca31ec366e4 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 24 Jan 2023 11:35:25 -0800 Subject: [PATCH 11/12] Update .github/workflows/common/common_release_workflow.yml Co-authored-by: Will Da Silva --- .github/workflows/common/common_release_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/common/common_release_workflow.yml b/.github/workflows/common/common_release_workflow.yml index 495a1d8..c5bdef9 100644 --- a/.github/workflows/common/common_release_workflow.yml +++ b/.github/workflows/common/common_release_workflow.yml @@ -17,7 +17,7 @@ name: Publish GitHub Release to PyPI (MeltanoLabs) # even if you are editing an existing draft. # - The text copy of Release Notes can freely be edited after publish, with no impact on the publish process and without # risk of accidentally republishing. -# - PyPi will only accept each semvar increment once, but failures to publish can be freely retried, and you can yank +# - PyPI will only accept each semvar increment once, but failures to publish can be freely retried, and you can yank # and release a patch on top if needed. on: From fe887b723aa8397f67878e426fd9dc36b7d51d4d Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 24 Jan 2023 11:43:47 -0800 Subject: [PATCH 12/12] remove --capture=no --- .github/workflows/common/common_pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/common/common_pytest.yml b/.github/workflows/common/common_pytest.yml index 9e09acc..a38c881 100644 --- a/.github/workflows/common/common_pytest.yml +++ b/.github/workflows/common/common_pytest.yml @@ -35,4 +35,5 @@ jobs: run: | poetry install - name: Test with pytest - run: poetry run pytest --capture=no + run: poetry run pytest +