build: Add CodSpeed

Signed-off-by: Dmitry Dygalo <[email protected]>
Stranger6667 · May 10, 2024 · fea1296 · fea1296
1 parent bcb6d39
commit fea1296
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 32 deletions.
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -0,0 +1,31 @@
+name: Benchmarks
+
+on:
+  push:
+    branches:
+      - "master"
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  rust:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: jsonschema
+
+      - run: cargo install cargo-codspeed
+
+      - run: cargo codspeed build
+        working-directory: ./jsonschema
+
+      - uses: CodSpeedHQ/action@v2
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          working-directory: ./jsonschema
diff --git a/bench_helpers/Cargo.toml b/bench_helpers/Cargo.toml
@@ -9,3 +9,4 @@ license = "MIT"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 criterion = { version = "0.5.1", features = [], default-features = false }
+codspeed-criterion-compat = "2.6.0"
diff --git a/bench_helpers/src/lib.rs b/bench_helpers/src/lib.rs
@@ -1,4 +1,4 @@
-use criterion::Criterion;
+use codspeed_criterion_compat::Criterion;
 use serde::Deserialize;
 use serde_json::{from_reader, Value};
 use std::{

diff --git a/jsonschema/Cargo.toml b/jsonschema/Cargo.toml
@@ -68,14 +68,15 @@ getrandom = { version = "0.2", features = ["js"] }
 
 [dev-dependencies]
 bench_helpers = { path = "../bench_helpers" }
+codspeed-criterion-compat = "2.6.0"
 criterion = { version = "0.5.1", features = [], default-features = false }
-lazy_static = "1.4"                                                               # Needed for json schema test suite
 json_schema_test_suite = { version = "0.3.0", path = "../jsonschema-test-suite" }
 jsonschema-valid = "0.5"
+lazy_static = "1.4"                                                               # Needed for json schema test suite
 mockito = "0.31"
 paste = "1.0"
 test-case = "3"
-valico = "3.6"
+valico = "4"
 
 # Benchmarks for `jsonschema`
 [[bench]]

diff --git a/jsonschema/benches/jsonschema.rs b/jsonschema/benches/jsonschema.rs
@@ -1,25 +1,25 @@
 use bench_helpers::{
     bench_citm, bench_fast, bench_geojson, bench_keywords, bench_openapi, bench_swagger,
 };
-use criterion::{criterion_group, criterion_main, Bencher, BenchmarkId, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, BenchmarkId, Criterion};
 use jsonschema::{paths::JsonPointerNode, JSONSchema};
 use serde_json::Value;
 
-macro_rules! jsonschema_rs_bench {
+macro_rules! jsonschema_bench {
     ($c:tt, $name:expr, $schema:ident, $instance:ident) => {{
         let compiled = JSONSchema::options()
             .with_meta_schemas()
             .compile(&$schema)
             .expect("Invalid schema");
         assert!(compiled.is_valid(&$instance), "Invalid instance");
         assert!(compiled.validate(&$instance).is_ok(), "Invalid instance");
-        $c.bench_function(&format!("{} jsonschema_rs/compile", $name), |b| {
+        $c.bench_function(&format!("{} jsonschema/compile", $name), |b| {
             b.iter(|| JSONSchema::options().with_meta_schemas().compile(&$schema))
         });
-        $c.bench_function(&format!("{} jsonschema_rs/is_valid", $name), |b| {
+        $c.bench_function(&format!("{} jsonschema/is_valid", $name), |b| {
             b.iter(|| compiled.is_valid(&$instance))
         });
-        $c.bench_function(&format!("{} jsonschema_rs/validate", $name), |b| {
+        $c.bench_function(&format!("{} jsonschema/validate", $name), |b| {
             b.iter(|| compiled.validate(&$instance).ok())
         });
     }};
@@ -28,33 +28,33 @@ macro_rules! jsonschema_rs_bench {
 fn large_schemas(c: &mut Criterion) {
     // Open API JSON Schema
     // Only `jsonschema` works correctly - other libraries do not recognize `zuora` as valid
-    bench_openapi(&mut |name, schema, instance| jsonschema_rs_bench!(c, name, schema, instance));
+    bench_openapi(&mut |name, schema, instance| jsonschema_bench!(c, name, schema, instance));
     // Swagger JSON Schema
-    bench_swagger(&mut |name, schema, instance| jsonschema_rs_bench!(c, name, schema, instance));
+    bench_swagger(&mut |name, schema, instance| jsonschema_bench!(c, name, schema, instance));
     // Canada borders in GeoJSON
-    bench_geojson(&mut |name, schema, instance| jsonschema_rs_bench!(c, name, schema, instance));
+    bench_geojson(&mut |name, schema, instance| jsonschema_bench!(c, name, schema, instance));
     // CITM catalog
-    bench_citm(&mut |name, schema, instance| jsonschema_rs_bench!(c, name, schema, instance));
+    bench_citm(&mut |name, schema, instance| jsonschema_bench!(c, name, schema, instance));
 }
 
 fn fast_schema(c: &mut Criterion) {
     bench_fast(&mut |name, schema, valid, invalid| {
         let compiled = JSONSchema::compile(&schema).expect("Valid schema");
         assert!(compiled.is_valid(&valid));
         assert!(!compiled.is_valid(&invalid));
-        c.bench_function(&format!("{} jsonschema_rs/compile", name), |b| {
+        c.bench_function(&format!("{} jsonschema/compile", name), |b| {
             b.iter(|| JSONSchema::compile(&schema).expect("Valid schema"))
         });
-        c.bench_function(&format!("{} jsonschema_rs/is_valid/valid", name), |b| {
+        c.bench_function(&format!("{} jsonschema/is_valid/valid", name), |b| {
             b.iter(|| compiled.is_valid(&valid))
         });
-        c.bench_function(&format!("{} jsonschema_rs/validate/valid", name), |b| {
+        c.bench_function(&format!("{} jsonschema/validate/valid", name), |b| {
             b.iter(|| compiled.validate(&valid).ok())
         });
-        c.bench_function(&format!("{} jsonschema_rs/is_valid/invalid", name), |b| {
+        c.bench_function(&format!("{} jsonschema/is_valid/invalid", name), |b| {
             b.iter(|| compiled.is_valid(&invalid))
         });
-        c.bench_function(&format!("{} jsonschema_rs/validate/invalid", name), |b| {
+        c.bench_function(&format!("{} jsonschema/validate/invalid", name), |b| {
             b.iter(|| {
                 let _: Vec<_> = compiled
                     .validate(&invalid)
@@ -75,7 +75,7 @@ fn keywords(c: &mut Criterion) {
         },
         &mut |c: &mut Criterion, name: &str, schema: &Value| {
             c.bench_with_input(
-                BenchmarkId::new(name, "jsonschema_rs/compile"),
+                BenchmarkId::new(name, "jsonschema/compile"),
                 schema,
                 |b, schema| {
                     b.iter(|| {
@@ -92,7 +92,7 @@ fn keywords(c: &mut Criterion) {
 fn validate_valid(c: &mut Criterion, name: &str, schema: &Value, instance: &Value) {
     let compiled = JSONSchema::compile(schema).expect("Valid schema");
     c.bench_with_input(
-        BenchmarkId::new(name, "jsonschema_rs/is_valid/valid"),
+        BenchmarkId::new(name, "jsonschema/is_valid/valid"),
         instance,
         |b, instance| {
             b.iter(|| {
@@ -101,7 +101,7 @@ fn validate_valid(c: &mut Criterion, name: &str, schema: &Value, instance: &Valu
         },
     );
     c.bench_with_input(
-        BenchmarkId::new(name, "jsonschema_rs/validate/valid"),
+        BenchmarkId::new(name, "jsonschema/validate/valid"),
         instance,
         |b, instance| {
             b.iter(|| {
@@ -114,7 +114,7 @@ fn validate_valid(c: &mut Criterion, name: &str, schema: &Value, instance: &Valu
 fn validate_invalid(c: &mut Criterion, name: &str, schema: &Value, instance: &Value) {
     let compiled = JSONSchema::compile(schema).expect("Valid schema");
     c.bench_with_input(
-        BenchmarkId::new(name, "jsonschema_rs/is_valid/invalid"),
+        BenchmarkId::new(name, "jsonschema/is_valid/invalid"),
         instance,
         |b, instance| {
             b.iter(|| {
@@ -123,7 +123,7 @@ fn validate_invalid(c: &mut Criterion, name: &str, schema: &Value, instance: &Va
         },
     );
     c.bench_with_input(
-        BenchmarkId::new(name, "jsonschema_rs/validate/invalid"),
+        BenchmarkId::new(name, "jsonschema/validate/invalid"),
         instance,
         |b, instance| {
             b.iter(|| {
@@ -162,11 +162,6 @@ fn json_pointer_node(c: &mut Criterion) {
     c.bench_with_input(BenchmarkId::new("jsonpointer", "big"), &node, bench);
 }
 
-criterion_group!(
-    arbitrary,
-    large_schemas,
-    fast_schema,
-    keywords,
-    json_pointer_node
-);
-criterion_main!(arbitrary);
+criterion_group!(common, large_schemas, fast_schema, json_pointer_node);
+criterion_group!(specific, keywords);
+criterion_main!(common, specific);
diff --git a/jsonschema/benches/jsonschema_valid.rs b/jsonschema/benches/jsonschema_valid.rs
@@ -1,5 +1,5 @@
 use bench_helpers::{bench_citm, bench_fast, bench_geojson, bench_keywords};
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use jsonschema_valid::schemas;
 use serde_json::Value;
 

diff --git a/jsonschema/benches/valico.rs b/jsonschema/benches/valico.rs
@@ -1,5 +1,5 @@
 use bench_helpers::{bench_citm, bench_fast, bench_geojson, bench_keywords, bench_swagger};
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use serde_json::Value;
 use valico::json_schema;