Skip to content

Commit

Permalink
test: Make criterion calculate transfer benchmark throughputs (#1738)
Browse files Browse the repository at this point in the history
* test: Make `criterion` calculate transfer benchmark throughputs

Like:
```
transfer/Run multiple transfers with varying seeds
                        time:   [213.61 ms 224.95 ms 235.29 ms]
                        thrpt:  [17.000 MiB/s 17.782 MiB/s 18.726 MiB/s]
                 change:
                        time:   [-17.204% -6.9690% +1.8324%] (p = 0.21 > 0.05)
                        thrpt:  [-1.7994% +7.4910% +20.778%]
                        No change in performance detected.
Found 5 outliers among 100 measurements (5.00%)
  5 (5.00%) low mild

transfer/Run multiple transfers with the same seed
                        time:   [175.41 ms 188.40 ms 201.32 ms]
                        thrpt:  [19.869 MiB/s 21.231 MiB/s 22.803 MiB/s]
                 change:
                        time:   [-5.9393% +5.5417% +18.399%] (p = 0.35 > 0.05)
                        thrpt:  [-15.540% -5.2507% +6.3144%]
                        No change in performance detected.
Found 14 outliers among 100 measurements (14.00%)
  8 (8.00%) low mild
  5 (5.00%) high mild
  1 (1.00%) high severe
  ```

* This doesn't actually run `cargo bench`, it runs `cargo flamegraph`
  • Loading branch information
larseggert authored Mar 16, 2024
1 parent fe2241e commit a68dd1c
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 9 deletions.
7 changes: 0 additions & 7 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,9 @@ jobs:
# Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
- name: Profile cargo bench transfer
run: |
# This re-runs part of the previous step, and would hence overwrite part of the criterion results.
# Avoid that by shuffling the directories around so this run uses its own results directory.
mv target/criterion target/criterion-bench
mv target/criterion-transfer-profile target/criterion || true
taskset -c 0 nice -n -20 \
cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
--bench --exact "Run multiple transfers with varying seeds" --noplot
# And now restore the directories.
mv target/criterion target/criterion-transfer-profile
mv target/criterion-bench target/criterion
- name: Profile client/server transfer
run: |
Expand Down
7 changes: 5 additions & 2 deletions neqo-transport/benches/transfer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

use std::time::Duration;

use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion};
use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion, Throughput};
use test_fixture::{
boxed,
sim::{
Expand All @@ -21,7 +21,9 @@ const JITTER: Duration = Duration::from_millis(10);
const TRANSFER_AMOUNT: usize = 1 << 22; // 4Mbyte

fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option<impl AsRef<str>>) {
c.bench_function(label, |b| {
let mut group = c.benchmark_group("transfer");
group.throughput(Throughput::Bytes(u64::try_from(TRANSFER_AMOUNT).unwrap()));
group.bench_function(label, |b| {
b.iter_batched(
|| {
let nodes = boxed![
Expand All @@ -44,6 +46,7 @@ fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option<impl AsRef<st
SmallInput,
)
});
group.finish();
}

fn benchmark_transfer_variable(c: &mut Criterion) {
Expand Down

0 comments on commit a68dd1c

Please sign in to comment.