catalogs.yml

name: Collect and publish catalogs
on:
  workflow_dispatch:
    inputs:
      channel:
        description: 'Channel to publish catalogs to'
        required: true
        default: stgn
  schedule:
    - cron: '5 * * * *' # Run every hour at HH:05

env:
  PIP_DISABLE_PIP_VERSION_CHECK: on
  PIP_DEFAULT_TIMEOUT: 10
  PIP_PROGRESS_BAR: off

jobs:
  catalog-aws:
    name: Collect AWS catalog
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.[aws]'
      - name: Collect catalog
        working-directory: src
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: python -m gpuhunt aws --output ../aws.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-aws
          path: aws.csv
          retention-days: 1

  catalog-azure:
    name: Collect Azure catalog
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@v4
      - uses: azure/login@v2
        with:
          creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.[azure]'
      - name: Collect catalog
        working-directory: src
        env:
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        run: python -m gpuhunt azure --output ../azure.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-azure
          path: azure.csv
          retention-days: 1

  catalog-cudo:
    name: Collect Cudo catalog
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.'
      - name: Collect catalog
        working-directory: src
        run: python -m gpuhunt cudo --output ../cudo.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-cudo
          path: cudo.csv
          retention-days: 1

  catalog-datacrunch:
    name: Collect DataCrunch catalog
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.[datacrunch]'
      - name: Collect catalog
        working-directory: src
        env:
          DATACRUNCH_CLIENT_ID: ${{ secrets.DATACRUNCH_CLIENT_ID }}
          DATACRUNCH_CLIENT_SECRET: ${{ secrets.DATACRUNCH_CLIENT_SECRET }}
        run: python -m gpuhunt datacrunch --output ../datacrunch.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-datacrunch
          path: datacrunch.csv
          retention-days: 1

  catalog-gcp:
    name: Collect GCP catalog
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider'
          service_account: 'dstack-gpu-pricing-ci@dstack.iam.gserviceaccount.com'
          create_credentials_file: true
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.[gcp]'
      - name: Collect catalog
        working-directory: src
        env:
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
        run: python -m gpuhunt gcp --output ../gcp.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-gcp
          path: gcp.csv
          retention-days: 1

  catalog-lambdalabs:
    name: Collect LambdaLabs catalog
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.'
      - name: Collect catalog
        working-directory: src
        env:
          LAMBDALABS_TOKEN: ${{ secrets.LAMBDALABS_TOKEN }}
        run: python -m gpuhunt lambdalabs --output ../lambdalabs.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-lambdalabs
          path: lambdalabs.csv
          retention-days: 1

  catalog-oci:
    name: Collect OCI catalog
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.[oci]'
      - name: Collect catalog
        working-directory: src
        run: python -m gpuhunt oci --output ../oci.csv
        env:
          OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }}
          OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }}
          OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }}
          OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }}
          OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }}
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-oci
          path: oci.csv
          retention-days: 1

  catalog-runpod:
    name: Collect Runpod catalog
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install -e '.'
      - name: Collect catalog
        working-directory: src
        run: python -m gpuhunt runpod --output ../runpod.csv
      - uses: actions/upload-artifact@v4
        with:
          name: catalogs-runpod
          path: runpod.csv
          retention-days: 1

  test-catalog:
    name: Test catalogs integrity
    needs:
      - catalog-aws
      - catalog-azure
      - catalog-cudo
      - catalog-datacrunch
      - catalog-gcp
      - catalog-lambdalabs
      - catalog-oci
      - catalog-runpod
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          pip install pip -U
          pip install '.[all]'
          pip install pytest
      - uses: actions/download-artifact@v4
        with:
          pattern: catalogs-*
          merge-multiple: true
      - name: Run integrity tests
        env:
          CATALOG_DIR: .
        run: pytest src/integrity_tests

  publish-catalog:
    name: Publish catalogs
    needs: [ test-catalog ]
    runs-on: ubuntu-latest
    steps:
      - name: Install AWS CLI
        run: pip install awscli
      - uses: actions/download-artifact@v4
        with:
          pattern: catalogs-*
          merge-multiple: true
      - name: Write version
        run: echo "$(date +%Y%m%d)-${{ github.run_number }}" > version
      - name: Package catalogs
        run: zip catalog.zip *.csv version
      - name: Set channel
        run: |
          if [[ ${{ github.event_name == 'workflow_dispatch' }} == true ]]; then
            CHANNEL=${{ inputs.channel }}
          else
            CHANNEL=${{ vars.CHANNEL }}
          fi
          echo "CHANNEL=$CHANNEL" >> $GITHUB_ENV
      - name: Upload to S3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          VERSION=$(cat version)
          aws s3 cp catalog.zip "s3://dstack-gpu-pricing/$CHANNEL/$VERSION/catalog.zip" --acl public-read
          cat version | aws s3 cp - "s3://dstack-gpu-pricing/$CHANNEL/version" --acl public-read
          aws s3 cp "s3://dstack-gpu-pricing/$CHANNEL/$VERSION/catalog.zip" "s3://dstack-gpu-pricing/$CHANNEL/latest/catalog.zip" --acl public-read