From ffa10531380cef9561e2f307ec82c54d209e25ca Mon Sep 17 00:00:00 2001
From: Michal Ondras <michal@coinlist.co>
Date: Tue, 12 Dec 2023 22:17:11 -0800
Subject: [PATCH] add workflows for publishing

---
 .github/workflows/_cache-dependencies.yml   | 45 +++++++++++
 .github/workflows/_publish-cl-data-diff.yml | 83 +++++++++++++++++++++
 .github/workflows/ci-cd-data-diff.yml       | 58 ++++++++++++++
 .github/workflows/ci.yml                    |  9 +--
 .github/workflows/ci_full.yml               |  8 +-
 .github/workflows/pull-requests-checks.yml  | 56 ++++++++++++++
 6 files changed, 244 insertions(+), 15 deletions(-)
 create mode 100644 .github/workflows/_cache-dependencies.yml
 create mode 100644 .github/workflows/_publish-cl-data-diff.yml
 create mode 100644 .github/workflows/ci-cd-data-diff.yml
 create mode 100644 .github/workflows/pull-requests-checks.yml

diff --git a/.github/workflows/_cache-dependencies.yml b/.github/workflows/_cache-dependencies.yml
new file mode 100644
index 000000000..9bc9a918b
--- /dev/null
+++ b/.github/workflows/_cache-dependencies.yml
@@ -0,0 +1,45 @@
+name: cache_dependencies
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        required: true
+        type: string
+      poetry_version:
+        required: true
+        type: string
+    outputs:
+      python_cache_key:
+        description: "The key of the primary cache of the python dependencies"
+        value: ${{ jobs.python-cache.outputs.key }}
+
+jobs:
+  python-cache:
+    runs-on: ubuntu-latest
+    outputs:
+      key: ${{ steps.define-cache-key.outputs.cache_key }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        id: setup-python
+        with:
+          python-version: '${{ inputs.python_version }}'
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ inputs.poetry_version }}
+          virtualenvs-in-project: true
+      - name: Define Cache Key
+        id: define-cache-key
+        run: |
+          echo "cache_key=python-${{ runner.os }}--${{ inputs.python_version }}-${{ inputs.poetry_version }}-${{ hashFiles('**/poetry.lock') }}" >> $GITHUB_OUTPUT
+      - name: Cache venv
+        id: cached-python
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: ${{ steps.define-cache-key.outputs.cache_key }}
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root
diff --git a/.github/workflows/_publish-cl-data-diff.yml b/.github/workflows/_publish-cl-data-diff.yml
new file mode 100644
index 000000000..6e420c71d
--- /dev/null
+++ b/.github/workflows/_publish-cl-data-diff.yml
@@ -0,0 +1,83 @@
+name: publish_data_diff
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        required: true
+        type: string
+      poetry_version:
+        required: true
+        type: string
+      python_cache_key:
+        required: true
+        type: string
+      role_to_assume:
+        required: true
+        type: string
+      aws_region:
+        required: true
+        type: string
+
+  permissions:
+    id-token: write
+    contents: read
+
+jobs:
+  build:
+    name: 'Publish cl-data-diff SDK'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ inputs.role_to_assume }}
+          aws-region: ${{ inputs.aws_region }}
+          mask-aws-account-id: 'no'
+
+      - uses: actions/checkout@v3
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '${{ env.PYTHON_VERSION }}'
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+          virtualenvs-in-project: true
+      - name: Restore cached key
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: .venv
+          key: ${{ inputs.python_cache_key }}
+
+      - name: Set env variables
+        run: |
+          echo $(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) > version.txt
+          echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV
+
+          echo "VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region us-west-2 list-package-versions --domain coinlist --repository cl-data-diff --format pypi --package cl-data --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count)" >> $GITHUB_ENV
+
+      - name: Publish dev version
+        if: ${{ github.ref_name != 'main' }}
+        run: |
+          echo "$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" > version.txt
+          echo "To install this wheel \`poetry add \"cl-data-diff==$(cat version.txt)[all]\" --source artifact\`"
+
+          poetry run toml set --toml-path python/pyproject.toml tool.poetry.version $(cat version.txt)
+
+          cd python
+          poetry build --format wheel
+          poetry publish --repository artifact --username aws --password $(aws codeartifact --region us-west-2 get-authorization-token --domain coinlist --query authorizationToken --output text)
+
+      - name: Publish new version
+        if: ${{ github.ref_name == 'main' }}
+        run: |
+          if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then
+              cd python
+              poetry build --format wheel
+              poetry publish --repository artifact --username aws --password $(aws codeartifact --region us-west-2 get-authorization-token --domain coinlist --query authorizationToken --output text)
+          fi
diff --git a/.github/workflows/ci-cd-data-diff.yml b/.github/workflows/ci-cd-data-diff.yml
new file mode 100644
index 000000000..5ab200c54
--- /dev/null
+++ b/.github/workflows/ci-cd-data-diff.yml
@@ -0,0 +1,58 @@
+name: CI CD pipeline
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write
+  contents: read
+  actions: write
+
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: styfle/cancel-workflow-action@0.12.0
+      with:
+        access_token: ${{ github.token }}
+
+  setup:
+    runs-on: ubuntu-latest
+    needs: [cancel]
+    outputs:
+      python_version: ${{ steps.set_var.outputs.python_version }}
+      poetry_version: ${{ steps.set_var.outputs.poetry_version }}
+    steps:
+      - id: set_var
+        run: |
+          echo "python_version=3.11" >> $GITHUB_OUTPUT
+          echo "poetry_version=1.7.1" >> $GITHUB_OUTPUT
+
+  cache-dependencies:
+    needs: [setup]
+    uses: ./.github/workflows/_cache-dependencies.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+
+    ci-cover-databases:
+    needs: [setup, cache-dependencies]
+    uses: ./.github/workflows/ci_full.yml
+
+  ci-cover-versions:
+    needs: [setup, cache-dependencies]
+    uses: ./.github/workflows/ci.yml
+
+  publish-sdk:
+    needs: [setup, cache-dependencies, ci-cover-databases, ci-cover-versions]
+    uses: ./.github/workflows/_publish-cl-data-diff.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+      python_cache_key: ${{ needs.cache-dependencies.outputs.python_cache_key }}
+      role_to_assume: "arn:aws:iam::141053534699:role/github-actions-cl-data-diff"
+      aws_region: "us-west-2"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fd995d5ab..018daf3c2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,14 +1,7 @@
 name: CI-COVER-VERSIONS
 
 on:
-#   push:
-#     paths:
-#       - '**.py'
-#       - '.github/workflows/**'
-#       - '!dev/**'
-  pull_request:
-    branches: [ master ]
-
+  worklow_call:
   workflow_dispatch:
 
 jobs:
diff --git a/.github/workflows/ci_full.yml b/.github/workflows/ci_full.yml
index d125d1142..d8915e0b7 100644
--- a/.github/workflows/ci_full.yml
+++ b/.github/workflows/ci_full.yml
@@ -1,13 +1,7 @@
 name: CI-COVER-DATABASES
 
 on:
-#   push:
-#     paths:
-#       - '**.py'
-#       - '.github/workflows/**'
-#       - '!dev/**'
-  pull_request:
-    branches: [ master ]
+  workflow_call:
   workflow_dispatch:
 
 permissions:
diff --git a/.github/workflows/pull-requests-checks.yml b/.github/workflows/pull-requests-checks.yml
new file mode 100644
index 000000000..7a02b961a
--- /dev/null
+++ b/.github/workflows/pull-requests-checks.yml
@@ -0,0 +1,56 @@
+name: PR checks
+
+on:
+  pull_request: {}
+
+permissions:
+  id-token: write
+  contents: read
+  actions: write
+
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: styfle/cancel-workflow-action@0.12.0
+      with:
+        access_token: ${{ github.token }}
+
+  setup:
+    runs-on: ubuntu-latest
+    needs: [cancel]
+    outputs:
+      python_version: ${{ steps.set_var.outputs.python_version }}
+      poetry_version: ${{ steps.set_var.outputs.poetry_version }}
+    steps:
+      - id: set_var
+        run: |
+          echo "python_version=3.11" >> $GITHUB_OUTPUT
+          echo "poetry_version=1.7.1" >> $GITHUB_OUTPUT
+
+  cache-dependencies:
+    needs: [setup]
+    uses: ./.github/workflows/_cache-dependencies.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+
+  ci-cover-databases:
+    needs: [setup, cache-dependencies]
+    uses: ./.github/workflows/ci_full.yml
+
+  ci-cover-versions:
+    needs: [setup, cache-dependencies]
+    uses: ./.github/workflows/ci.yml
+
+  publish-sdk:
+    needs: [setup, cache-dependencies, cl-cover-databases,cl-cover-versions]
+    uses: ./.github/workflows/_publish-cl-data-diff.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+      python_cache_key: ${{ needs.cache-dependencies.outputs.python_cache_key }}
+      role_to_assume: "arn:aws:iam::141053534699:role/github-actions-cl-data-diff"
+      aws_region: "us-west-2"