diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml
index 7c78218fc..c3d4afcc0 100644
--- a/.github/workflows/dry-run.yml
+++ b/.github/workflows/dry-run.yml
@@ -42,7 +42,8 @@ jobs:
           rustup default stable
           rustc -vV
 
-      - uses: Swatinem/rust-cache@v2
+      # Pinning a specific version to avoid surprises
+      - uses: Swatinem/rust-cache@27b8ea9368cf428f0bfe41b0876b1a7e809d9844
 
       - name: Download built JSON API
         uses: actions/download-artifact@v4
@@ -134,8 +135,8 @@ jobs:
         run: |
           # Perform build and execution separately to avoid any potential output from
           # cargo leaking into the output file.
-          cargo build
-          ./target/debug/sync-team print-plan --services github --team-json team-api > output.txt
+          cargo build --release
+          ./target/release/sync-team print-plan --services github --team-json team-api > output.txt
 
       - name: Prepare comment
         run: |
@@ -149,11 +150,24 @@ jobs:
 
           cat comment.txt
 
+      - name: Extract PR number
+        run: |
+          # We read the PR number that is stored in the uploaded archive
+          # and check that it is an integer (as the workflow could upload whatever it wants).
+          UNSANITIZED_PR=`cat team-api/pr.txt`
+          if [[ ${UNSANITIZED_PR} =~ ^[0-9]+$ ]]; then
+            echo "PR_NUMBER=${UNSANITIZED_PR}" >> $GITHUB_ENV
+          else
+            echo "Invalid PR number passed: ${UNSANITIZED_PR}"
+            exit 1
+          fi
+
       - name: Send comment
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          PR=${{ github.event.workflow_run.pull_requests[0].number }}
+          PR=${PR_NUMBER}
+          echo "Pull request ${PR}"
 
           # --edit-last doesn't work if there is no previous comment, so we have to figure out
           # if we should create a comment or not
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 933fdafab..632a06202 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,6 +43,10 @@ jobs:
           cargo run -- static-api build
           echo "team-api.infra.rust-lang.org" > build/CNAME
 
+      - name: Write PR number into the uploaded archive
+        if: ${{ github.event_name == 'pull_request' }}
+        run: echo "${{ github.event.pull_request.number }}" > build/pr.txt
+
       - name: Upload the built JSON as a GitHub artifact
         if: ${{ github.event_name == 'pull_request' }}
         uses: actions/upload-artifact@v4