Use Codspeed for continous benchmarking (#6896)

.github/workflows/ci.yaml

@@ -341,3 +341,28 @@ jobs:
         run: cat target/progress_projects_stats.txt > $GITHUB_STEP_SUMMARY
       - name: "Remove checkouts from cache"
         run: rm -r target/progress_projects
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@v3
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: "Build benchmarks"
+        run: cargo codspeed build --features codspeed -p ruff_benchmark
+
+      - name: "Run benchmarks"
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}

.github/workflows/pr-comment.yaml

@@ -2,7 +2,7 @@ name: PR Check Comment
 
 on:
   workflow_run:
-    workflows: [CI, Benchmark]
+    workflows: [CI]
     types: [completed]
   workflow_dispatch:
     inputs:
@@ -43,35 +43,16 @@ jobs:
           path: pr/ecosystem
           if_no_artifact_found: ignore
 
-      - uses: dawidd6/action-download-artifact@v2
-        name: "Download Benchmark Result"
-        id: download-benchmark-result
-        if: steps.pr-number.outputs.pr-number
-        with:
-          name: summary
-          workflow: benchmark.yaml
-          pr: ${{ steps.pr-number.outputs.pr-number }}
-          path: pr/benchmark
-          if_no_artifact_found: ignore
-
       - name: Generate Comment
         id: generate-comment
-        if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true'
+        if: steps.download-ecosystem-result.outputs.found_artifact == 'true'
         run: |
           echo 'comment<<EOF' >> $GITHUB_OUTPUT
           echo '## PR Check Results' >> $GITHUB_OUTPUT
 
-          if [[ -f pr/ecosystem/ecosystem-result ]]
-          then
-            echo "### Ecosystem" >> $GITHUB_OUTPUT
-            cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
-            echo "" >> $GITHUB_OUTPUT
-          fi
-
-          if [[ -f pr/benchmark/summary.md ]]
-          then
-            cat pr/benchmark/summary.md >> $GITHUB_OUTPUT
-          fi
+          echo "### Ecosystem" >> $GITHUB_OUTPUT
+          cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
+          echo "" >> $GITHUB_OUTPUT
 
           echo 'EOF' >> $GITHUB_OUTPUT
 

crates/ruff_benchmark/Cargo.toml

@@ -32,14 +32,18 @@ serde.workspace = true
 serde_json.workspace = true
 url = "2.3.1"
 ureq = "2.6.2"
+criterion = { version = "0.5.1"}
+codspeed-criterion-compat = { version="2.1.0", optional = true}
 
 [dev-dependencies]
 ruff.path = "../ruff"
 ruff_python_ast.path = "../ruff_python_ast"
 ruff_python_formatter = { path = "../ruff_python_formatter" }
 ruff_python_index = { path = "../ruff_python_index" }
 ruff_python_parser = { path = "../ruff_python_parser" }
-criterion = { version = "0.5.1"}
+
+[features]
+codspeed = ["codspeed-criterion-compat"]
 
 [target.'cfg(target_os = "windows")'.dev-dependencies]
 mimalloc = "0.1.34"

crates/ruff_benchmark/benches/formatter.rs

@@ -1,8 +1,9 @@
 use std::path::Path;
 use std::time::Duration;
 
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
-
+use ruff_benchmark::criterion::{
+    criterion_group, criterion_main, BenchmarkId, Criterion, Throughput,
+};
 use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_node, PyFormatOptions};
 use ruff_python_index::CommentRangesBuilder;

crates/ruff_benchmark/benches/linter.rs

@@ -1,14 +1,12 @@
 use std::time::Duration;
 
-use criterion::measurement::WallTime;
-use criterion::{
-    criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
-};
-
 use ruff::linter::lint_only;
 use ruff::settings::{flags, Settings};
 use ruff::source_kind::SourceKind;
 use ruff::RuleSelector;
+use ruff_benchmark::criterion::{
+    criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
+};
 use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
 use ruff_python_ast::PySourceType;
 
@@ -43,7 +41,7 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
     ])
 }
 
-fn benchmark_linter(mut group: BenchmarkGroup<WallTime>, settings: &Settings) {
+fn benchmark_linter(mut group: BenchmarkGroup, settings: &Settings) {
     let test_cases = create_test_cases().unwrap();
 
     for case in test_cases {

crates/ruff_benchmark/benches/parser.rs

@@ -1,7 +1,8 @@
 use std::time::Duration;
 
-use criterion::measurement::WallTime;
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use ruff_benchmark::criterion::{
+    criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
+};
 use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;

crates/ruff_benchmark/src/criterion.rs

@@ -0,0 +1,11 @@
+//! This module re-exports the criterion API but picks the right backend depending on whether
+//! the benchmarks are built to run locally or with codspeed
+
+#[cfg(not(codspeed))]
+pub use criterion::*;
+
+#[cfg(not(codspeed))]
+pub type BenchmarkGroup<'a> = criterion::BenchmarkGroup<'a, measurement::WallTime>;
+
+#[cfg(codspeed)]
+pub use codspeed_criterion_compat::*;

crates/ruff_benchmark/src/lib.rs

@@ -1,3 +1,5 @@
+pub mod criterion;
+
 use std::fmt::{Display, Formatter};
 use std::path::PathBuf;
 use std::process::Command;