Merge branch 'master' into dependabot/pip/black-24.3.0

getsentry · Nov 1, 2024 · c47db35 · c47db35
2 parents 10ca454 + b9e330a
commit c47db35
Show file tree

Hide file tree

Showing 713 changed files with 56,427 additions and 18,844 deletions.
diff --git a/.craft.yml b/.craft.yml
@@ -10,11 +10,11 @@ statusProvider:
 targets:
   - id: release
     name: docker
-    source: us.gcr.io/sentryio/snuba
+    source: us-central1-docker.pkg.dev/sentryio/snuba/image
     target: getsentry/snuba
   - id: latest
     name: docker
-    source: us.gcr.io/sentryio/snuba
+    source: us-central1-docker.pkg.dev/sentryio/snuba/image
     target: getsentry/snuba
     targetFormat: '{{{target}}}:latest'
   - name: github
diff --git a/.gitattributes b/.gitattributes
@@ -1 +1,3 @@
 snuba/admin/dist/bundle.js binary
+snuba/protobufs/**/*.py linguist-generated=true
+snuba/protobufs/**/*.pyi linguist-generated=true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -76,7 +76,7 @@ jobs:
       # If working tree is dirty, commit and update if we have a token
       - name: Apply any pre-commit fixed files
         if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always()
-        uses: getsentry/action-github-commit@v1.0.0
+        uses: getsentry/action-github-commit@v2.1.0
         with:
           github-token: ${{ steps.token.outputs.token }}
 
@@ -153,7 +153,7 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
       - name: Build snuba docker image for CI
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         if: github.repository_owner == 'getsentry'
         with:
           context: .
@@ -199,48 +199,36 @@ jobs:
             "test_distributed_migrations",
           ]
     steps:
-      - uses: actions/checkout@v2
-        name: Checkout code
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-      - name: Build snuba docker image for CI
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: false
-          tags: snuba-test
-          load: true
-          target: testing
-          cache-from: |
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
+      - name: Registry login
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
       - name: Docker set up
         run: |
           docker network create --attachable cloudbuild
 
       - name: Docker Snuba Rust tests
         run: |
-          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test-rust
+          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test-rust
         if: ${{ matrix.snuba_settings == 'test_rust' }}
 
       - name: Docker Snuba tests
         run: |
-          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker-compose -f docker-compose.gcb.yml run --rm snuba-test
+          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker compose -f docker-compose.gcb.yml run --rm snuba-test
         if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings  == 'test_distributed' }}
 
       - name: Docker Snuba Multi-Node Tests
         run: |
-          SNUBA_SETTINGS=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml up -d
-          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test
+          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml up -d
+          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test
         if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }}
 
       - name: Docker Snuba Init Tests
         run: |
-          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test
+          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker compose -f docker-compose.gcb.yml run --rm snuba-test
         if: ${{ matrix.snuba_settings == 'test' }}
 
       - name: Upload test results to Codecov
@@ -281,28 +269,13 @@ jobs:
       TEST_GROUP_STRATEGY: ROUND_ROBIN
 
     steps:
-      # Checkout codebase
-      - name: Checkout snuba
+      - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build snuba docker image for CI
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: false
-          tags: snuba-test
-          load: true
-          target: testing
-          cache-from: |
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
+      - name: Registry login
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
-      # Checkout Sentry and run integration tests against latest snuba
-      # Make sure this is after `docker build`, otherwise we'll break docker cache
       - name: Checkout sentry
         uses: actions/checkout@v2
         with:
@@ -342,7 +315,7 @@ jobs:
             -e REDIS_DB=1 \
             --name sentry_snuba \
             --network sentry \
-            snuba-test
+            ghcr.io/getsentry/snuba-ci:${{ github.sha }}
           docker exec sentry_snuba snuba migrations migrate --force
 
       - name: Run snuba tests
@@ -366,8 +339,12 @@ jobs:
               tests/sentry/search/events \
               tests/sentry/event_manager \
               tests/sentry/api/endpoints/test_organization_profiling_functions.py \
+              tests/snuba/api/endpoints/test_organization_events_stats_mep.py \
+              tests/sentry/sentry_metrics/querying \
               tests/snuba/test_snql_snuba.py \
               tests/snuba/test_metrics_layer.py \
+              tests/sentry/api/endpoints/test_organization_spans_fields.py \
+              tests/snuba/api/endpoints/test_organization_events_span_indexed.py \
               -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"
 
       - name: Run CI module tests
@@ -385,31 +362,16 @@ jobs:
       matrix:
         version:
           [
-            "21.8.13.1.altinitystable",
-            "22.8.15.25.altinitystable",
-            "23.3.13.7.altinitystable",
-            "23.8.8.21.altinitystable",
+            "23.8.11.29.altinitystable",
           ]
 
     steps:
-      - uses: actions/checkout@v2
-        name: Checkout code
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-      - name: Build snuba docker image for CI
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: false
-          tags: snuba-test
-          load: true
-          target: testing
-          cache-from: |
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
-            type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
+      - name: Registry login
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
       - name: Docker set up
         run: |
@@ -418,11 +380,27 @@ jobs:
       - name: Docker Snuba Test other ClickHouse versions
         run: |
           export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}}
-          SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test
+          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test
 
       - name: Upload to codecov
         run: |
           curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}
+
+  self-hosted-end-to-end:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout Snuba
+        uses: actions/checkout@v3
+      - name: Run Sentry self-hosted e2e CI
+        uses: getsentry/action-self-hosted-e2e-tests@main
+        with:
+          project_name: snuba
+          docker_repo: getsentry/snuba
+          image_url: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.event.pull_request.head.sha || github.sha }}
+          docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }}
+
   publish-to-dockerhub:
     name: Publish Snuba to DockerHub
     runs-on: ubuntu-20.04
@@ -432,7 +410,7 @@ jobs:
       - name: Pull the test image
         id: image_pull
         env:
-          IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }}
+          IMAGE_URL: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.sha }}
         shell: bash
         run: |
           echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out."
@@ -453,7 +431,7 @@ jobs:
         shell: bash
         env:
           SHORT_SHA: ${{ steps.short_sha.outputs.sha }}
-          IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }}
+          IMAGE_URL: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.sha }}
         run: |
           # only login if the password is set
           if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" | docker login --username=sentrybuilder --password-stdin; fi

diff --git a/.github/workflows/ddl-changes.yml b/.github/workflows/ddl-changes.yml
@@ -22,7 +22,7 @@ jobs:
           fetch-depth: 200
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -13,7 +13,7 @@ jobs:
       - name: "Checkout Repository"
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Dependency Review
-        uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 # v3.1.0
+        uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0
         with:
           # Possible values: "critical", "high", "moderate", "low"
           fail-on-severity: high
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -30,7 +30,7 @@ jobs:
           force_orphan: true
 
       - name: Archive Docs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: docs
           path: docs/build
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,8 @@
 exclude: >
     (?x)(
         ^snuba/admin/dist/|
-        ^snuba/admin/static/
+        ^snuba/admin/static/|
+        ^snuba/protobufs/
     )
 
 repos:

diff --git a/.python-version b/.python-version
@@ -1 +1 @@
-3.11.6
+3.11.8
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -1,6 +1,13 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        },
         {
             "name": "Pytest Current File",
             "type": "debugpy",
@@ -10,7 +17,6 @@
                 "${file}"
             ],
             "console": "integratedTerminal",
-            "justMyCode": true  // wasn't sure if I should keep or delete
         }
     ]
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -40,7 +40,7 @@
 
   "[typescriptreact]": {
     "editor.formatOnSave": true,
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "vscode.typescript-language-features"
   },
 
   "[less]": {
@@ -57,13 +57,14 @@
   },
 
   "python.linting.mypyEnabled": true,
-  "python.linting.mypyArgs": ["--strict"],
   "python.linting.flake8Enabled": true,
   "python.formatting.provider": "black",
   // https://github.com/DonJayamanne/pythonVSCode/issues/992
   "python.pythonPath": ".venv/bin/python",
   // test discovery is sluggish and the UI around running
   // tests is often in your way and misclicked
   "python.testing.pytestEnabled": true,
-  "python.testing.pytestArgs": ["tests"]
+  "python.testing.pytestArgs": ["tests"],
+  "python.analysis.autoImportCompletions": true,
+  "mypy-type-checker.args": ["--strict"]
 }