Collect and publish catalogs #3486
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Collect and publish catalogs | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| channel: | |
| description: 'Channel to publish catalogs to' | |
| required: true | |
| default: stgn | |
| schedule: | |
| - cron: '5 * * * *' # Run every hour at HH:05 | |
| env: | |
| PIP_DISABLE_PIP_VERSION_CHECK: on | |
| PIP_DEFAULT_TIMEOUT: 10 | |
| PIP_PROGRESS_BAR: off | |
| jobs: | |
| catalog-aws: | |
| name: Collect AWS catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[aws]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: python -m gpuhunt aws --output ../aws.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-aws | |
| path: aws.csv | |
| retention-days: 1 | |
| catalog-azure: | |
| name: Collect Azure catalog | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: azure/login@v2 | |
| with: | |
| creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[azure]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: python -m gpuhunt azure --output ../azure.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-azure | |
| path: azure.csv | |
| retention-days: 1 | |
| catalog-cudo: | |
| name: Collect Cudo catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: python -m gpuhunt cudo --output ../cudo.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-cudo | |
| path: cudo.csv | |
| retention-days: 1 | |
| catalog-datacrunch: | |
| name: Collect DataCrunch catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[datacrunch]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| DATACRUNCH_CLIENT_ID: ${{ secrets.DATACRUNCH_CLIENT_ID }} | |
| DATACRUNCH_CLIENT_SECRET: ${{ secrets.DATACRUNCH_CLIENT_SECRET }} | |
| run: python -m gpuhunt datacrunch --output ../datacrunch.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-datacrunch | |
| path: datacrunch.csv | |
| retention-days: 1 | |
| catalog-gcp: | |
| name: Collect GCP catalog | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' | |
| service_account: '[email protected]' | |
| create_credentials_file: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[gcp]' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| run: python -m gpuhunt gcp --output ../gcp.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-gcp | |
| path: gcp.csv | |
| retention-days: 1 | |
| catalog-lambdalabs: | |
| name: Collect LambdaLabs catalog | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.' | |
| - name: Collect catalog | |
| working-directory: src | |
| env: | |
| LAMBDALABS_TOKEN: ${{ secrets.LAMBDALABS_TOKEN }} | |
| run: python -m gpuhunt lambdalabs --output ../lambdalabs.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-lambdalabs | |
| path: lambdalabs.csv | |
| retention-days: 1 | |
| catalog-oci: | |
| name: Collect OCI catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.[oci]' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: python -m gpuhunt oci --output ../oci.csv | |
| env: | |
| OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} | |
| OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} | |
| OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} | |
| OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} | |
| OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-oci | |
| path: oci.csv | |
| retention-days: 1 | |
| catalog-runpod: | |
| name: Collect Runpod catalog | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install -e '.' | |
| - name: Collect catalog | |
| working-directory: src | |
| run: python -m gpuhunt runpod --output ../runpod.csv | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: catalogs-runpod | |
| path: runpod.csv | |
| retention-days: 1 | |
| test-catalog: | |
| name: Test catalogs integrity | |
| needs: | |
| - catalog-aws | |
| - catalog-azure | |
| - catalog-cudo | |
| - catalog-datacrunch | |
| - catalog-gcp | |
| - catalog-lambdalabs | |
| - catalog-oci | |
| - catalog-runpod | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: | | |
| pip install pip -U | |
| pip install '.[all]' | |
| pip install pytest | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: catalogs-* | |
| merge-multiple: true | |
| - name: Run integrity tests | |
| env: | |
| CATALOG_DIR: . | |
| run: pytest src/integrity_tests | |
| publish-catalog: | |
| name: Publish catalogs | |
| needs: [ test-catalog ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install AWS CLI | |
| run: pip install awscli | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: catalogs-* | |
| merge-multiple: true | |
| - name: Write version | |
| run: echo "$(date +%Y%m%d)-${{ github.run_number }}" > version | |
| - name: Package catalogs | |
| run: zip catalog.zip *.csv version | |
| - name: Set channel | |
| run: | | |
| if [[ ${{ github.event_name == 'workflow_dispatch' }} == true ]]; then | |
| CHANNEL=${{ inputs.channel }} | |
| else | |
| CHANNEL=${{ vars.CHANNEL }} | |
| fi | |
| echo "CHANNEL=$CHANNEL" >> $GITHUB_ENV | |
| - name: Upload to S3 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| VERSION=$(cat version) | |
| aws s3 cp catalog.zip "s3://dstack-gpu-pricing/$CHANNEL/$VERSION/catalog.zip" --acl public-read | |
| cat version | aws s3 cp - "s3://dstack-gpu-pricing/$CHANNEL/version" --acl public-read | |
| aws s3 cp "s3://dstack-gpu-pricing/$CHANNEL/$VERSION/catalog.zip" "s3://dstack-gpu-pricing/$CHANNEL/latest/catalog.zip" --acl public-read |