diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml deleted file mode 100644 index b891e1d4fd5fe..0000000000000 --- a/.github/workflows/benchmark.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: Benchmark - -on: - pull_request: - paths: - - "Cargo.toml" - - "Cargo.lock" - - "rust-toolchain" - - "crates/**" - - "!crates/ruff_dev" - - "!crates/ruff_shrinking" - - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -jobs: - run-benchmark: - if: github.event_name == 'pull_request' - name: "Run | ${{ matrix.os }}" - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - name: "PR - Checkout Branch" - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "PR - Install Rust toolchain" - run: rustup show - - - uses: Swatinem/rust-cache@v2 - - - name: "PR - Build benchmarks" - run: cargo bench -p ruff_benchmark --no-run - - - name: "PR - Run benchmarks" - run: cargo benchmark --save-baseline=pr - - - name: "Main - Checkout Branch" - uses: actions/checkout@v3 - with: - clean: false - ref: main - - - name: "Main - Install Rust toolchain" - run: rustup show - - - name: "Main - Build benchmarks" - run: cargo bench -p ruff_benchmark --no-run - - - name: "Main - Run benchmarks" - run: cargo benchmark --save-baseline=main - - - name: "Upload benchmark results" - uses: actions/upload-artifact@v3 - with: - name: benchmark-results-${{ matrix.os }} - path: ./target/criterion - - # Cleanup - - name: Remove Criterion Artifact - uses: JesseTG/rm@v1.0.3 - with: - path: ./target/criterion - - benchmark-compare: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - name: Compare - needs: - - run-benchmark - - steps: - - name: "Install Rust toolchain" - run: rustup show - - - name: "Install critcmp" - uses: taiki-e/install-action@v2 - with: - tool: critcmp - - - name: "Linux | Download PR benchmark results" - uses: actions/download-artifact@v3 - with: - name: benchmark-results-ubuntu-latest - path: ./target/criterion - - - name: "Linux | Compare benchmark results" - shell: bash - run: | - echo "### Benchmark" >> summary.md - echo "#### Linux" >> summary.md - echo "\`\`\`" >> summary.md - critcmp main pr >> summary.md - echo "\`\`\`" >> summary.md - echo "" >> summary.md - - - name: "Linux | Cleanup benchmark results" - run: rm -rf ./target/criterion - - - name: "Windows | Download PR benchmark results" - uses: actions/download-artifact@v3 - with: - name: benchmark-results-windows-latest - path: ./target/criterion - - - name: "Windows | Compare benchmark results" - shell: bash - run: | - echo "#### Windows" >> summary.md - echo "\`\`\`" >> summary.md - critcmp main pr >> summary.md - echo "\`\`\`" >> summary.md - echo "" >> summary.md - - echo ${{ github.event.pull_request.number }} > pr-number - - cat summary.md > $GITHUB_STEP_SUMMARY - - - uses: actions/upload-artifact@v3 - name: Upload PR Number - with: - name: pr-number - path: pr-number - - - uses: actions/upload-artifact@v3 - name: Upload Summary - with: - name: summary - path: summary.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b2681cec59f80..0a044511bf465 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -341,3 +341,28 @@ jobs: run: cat target/progress_projects_stats.txt > $GITHUB_STEP_SUMMARY - name: "Remove checkouts from cache" run: rm -r target/progress_projects + + benchmarks: + runs-on: ubuntu-latest + steps: + - name: "Checkout Branch" + uses: actions/checkout@v3 + + - name: "Install Rust toolchain" + run: rustup show + + - name: "Install codspeed" + uses: taiki-e/install-action@v2 + with: + tool: cargo-codspeed + + - uses: Swatinem/rust-cache@v2 + + - name: "Build benchmarks" + run: cargo codspeed build --features codspeed -p ruff_benchmark + + - name: "Run benchmarks" + uses: CodSpeedHQ/action@v1 + with: + run: cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/pr-comment.yaml b/.github/workflows/pr-comment.yaml index cd2070f2fcf85..1e0327d68e44f 100644 --- a/.github/workflows/pr-comment.yaml +++ b/.github/workflows/pr-comment.yaml @@ -2,7 +2,7 @@ name: PR Check Comment on: workflow_run: - workflows: [CI, Benchmark] + workflows: [CI] types: [completed] workflow_dispatch: inputs: @@ -43,35 +43,16 @@ jobs: path: pr/ecosystem if_no_artifact_found: ignore - - uses: dawidd6/action-download-artifact@v2 - name: "Download Benchmark Result" - id: download-benchmark-result - if: steps.pr-number.outputs.pr-number - with: - name: summary - workflow: benchmark.yaml - pr: ${{ steps.pr-number.outputs.pr-number }} - path: pr/benchmark - if_no_artifact_found: ignore - - name: Generate Comment id: generate-comment - if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true' + if: steps.download-ecosystem-result.outputs.found_artifact == 'true' run: | echo 'comment<> $GITHUB_OUTPUT echo '## PR Check Results' >> $GITHUB_OUTPUT - if [[ -f pr/ecosystem/ecosystem-result ]] - then - echo "### Ecosystem" >> $GITHUB_OUTPUT - cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - fi - - if [[ -f pr/benchmark/summary.md ]] - then - cat pr/benchmark/summary.md >> $GITHUB_OUTPUT - fi + echo "### Ecosystem" >> $GITHUB_OUTPUT + cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT diff --git a/Cargo.lock b/Cargo.lock index 261438324222e..476b9192094e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -414,6 +414,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "codspeed" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeec2fbed4969dc38b5ca201115dd5c2614b8ef78e0a7221dd5f0977fb1552b" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b13f0a08d40ce7c95bdf288f725b975e62fcadfa8ba152340943bab6de43af7" +dependencies = [ + "codspeed", + "colored", + "criterion", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -2138,6 +2160,7 @@ dependencies = [ name = "ruff_benchmark" version = "0.0.0" dependencies = [ + "codspeed-criterion-compat", "criterion", "mimalloc", "once_cell", @@ -2696,9 +2719,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.100" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", diff --git a/crates/ruff_benchmark/Cargo.toml b/crates/ruff_benchmark/Cargo.toml index 1d2baf66ddf35..5753044d6c78f 100644 --- a/crates/ruff_benchmark/Cargo.toml +++ b/crates/ruff_benchmark/Cargo.toml @@ -32,6 +32,8 @@ serde.workspace = true serde_json.workspace = true url = "2.3.1" ureq = "2.6.2" +criterion = { version = "0.5.1"} +codspeed-criterion-compat = { version="2.1.0", optional = true} [dev-dependencies] ruff.path = "../ruff" @@ -39,7 +41,9 @@ ruff_python_ast.path = "../ruff_python_ast" ruff_python_formatter = { path = "../ruff_python_formatter" } ruff_python_index = { path = "../ruff_python_index" } ruff_python_parser = { path = "../ruff_python_parser" } -criterion = { version = "0.5.1"} + +[features] +codspeed = ["codspeed-criterion-compat"] [target.'cfg(target_os = "windows")'.dev-dependencies] mimalloc = "0.1.34" diff --git a/crates/ruff_benchmark/benches/formatter.rs b/crates/ruff_benchmark/benches/formatter.rs index 75f7298f4bae0..cc52e167c039d 100644 --- a/crates/ruff_benchmark/benches/formatter.rs +++ b/crates/ruff_benchmark/benches/formatter.rs @@ -1,8 +1,9 @@ use std::path::Path; use std::time::Duration; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; - +use ruff_benchmark::criterion::{ + criterion_group, criterion_main, BenchmarkId, Criterion, Throughput, +}; use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError}; use ruff_python_formatter::{format_node, PyFormatOptions}; use ruff_python_index::CommentRangesBuilder; diff --git a/crates/ruff_benchmark/benches/linter.rs b/crates/ruff_benchmark/benches/linter.rs index 796c96d21ab00..f48b55b7d7a09 100644 --- a/crates/ruff_benchmark/benches/linter.rs +++ b/crates/ruff_benchmark/benches/linter.rs @@ -1,14 +1,12 @@ use std::time::Duration; -use criterion::measurement::WallTime; -use criterion::{ - criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput, -}; - use ruff::linter::lint_only; use ruff::settings::{flags, Settings}; use ruff::source_kind::SourceKind; use ruff::RuleSelector; +use ruff_benchmark::criterion::{ + criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput, +}; use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError}; use ruff_python_ast::PySourceType; @@ -43,7 +41,7 @@ fn create_test_cases() -> Result, TestFileDownloadError> { ]) } -fn benchmark_linter(mut group: BenchmarkGroup, settings: &Settings) { +fn benchmark_linter(mut group: BenchmarkGroup, settings: &Settings) { let test_cases = create_test_cases().unwrap(); for case in test_cases { diff --git a/crates/ruff_benchmark/benches/parser.rs b/crates/ruff_benchmark/benches/parser.rs index fa3ed713266d4..4efbb72f3c43a 100644 --- a/crates/ruff_benchmark/benches/parser.rs +++ b/crates/ruff_benchmark/benches/parser.rs @@ -1,7 +1,8 @@ use std::time::Duration; -use criterion::measurement::WallTime; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use ruff_benchmark::criterion::{ + criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput, +}; use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError}; use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor}; use ruff_python_ast::Stmt; diff --git a/crates/ruff_benchmark/src/criterion.rs b/crates/ruff_benchmark/src/criterion.rs new file mode 100644 index 0000000000000..213403c1cec5f --- /dev/null +++ b/crates/ruff_benchmark/src/criterion.rs @@ -0,0 +1,11 @@ +//! This module re-exports the criterion API but picks the right backend depending on whether +//! the benchmarks are built to run locally or with codspeed + +#[cfg(not(codspeed))] +pub use criterion::*; + +#[cfg(not(codspeed))] +pub type BenchmarkGroup<'a> = criterion::BenchmarkGroup<'a, measurement::WallTime>; + +#[cfg(codspeed)] +pub use codspeed_criterion_compat::*; diff --git a/crates/ruff_benchmark/src/lib.rs b/crates/ruff_benchmark/src/lib.rs index 70d2e7a34f34d..b5e60cd3b420f 100644 --- a/crates/ruff_benchmark/src/lib.rs +++ b/crates/ruff_benchmark/src/lib.rs @@ -1,3 +1,5 @@ +pub mod criterion; + use std::fmt::{Display, Formatter}; use std::path::PathBuf; use std::process::Command;