Squashed commit of the following:

commit 9ca5ecc Merge: f50f414 f408321 Author: Lars Eggert <[email protected]> Date: Fri Mar 15 00:12:29 2024 +0200 Merge branch 'main' into ci-bench-cc Signed-off-by: Lars Eggert <[email protected]> commit f50f414 Merge: 8e5290b bc262a5 Author: Lars Eggert <[email protected]> Date: Wed Mar 13 07:59:01 2024 +0200 Merge branch 'main' into ci-bench-cc commit 8e5290b Merge: f0cd19e 2ff9742 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 22:42:54 2024 +0200 Merge branch 'main' into ci-bench-cc commit f0cd19e Merge: b2bb855 17c4175 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 21:54:08 2024 +0200 Merge branch 'main' into ci-bench-cc commit b2bb855 Merge: d072504 4ea2c56 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 17:25:13 2024 +0200 Merge branch 'ci-bench-cc' of github.com:larseggert/neqo into ci-bench-cc commit d072504 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 17:24:52 2024 +0200 Reorder things so `results.ms` is included in the exported artifact commit 4ea2c56 Merge: c82ff3a 5c72890 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 17:18:37 2024 +0200 Merge branch 'main' into ci-bench-cc commit c82ff3a Author: Lars Eggert <[email protected]> Date: Tue Mar 12 16:41:59 2024 +0200 `killall` -> `pkill` commit d37e706 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 16:37:50 2024 +0200 Go back to `killall` commit 11320d0 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 16:11:38 2024 +0200 No -INT commit 407bd4f Author: Lars Eggert <[email protected]> Date: Tue Mar 12 14:33:52 2024 +0200 kill -> killall Also reduce test transfer size. commit 9d3a8b7 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 13:57:51 2024 +0200 Use temp dir, and fix path error commit 84e2206 Merge: 925cc12 b0d816a Author: Lars Eggert <[email protected]> Date: Tue Mar 12 11:10:41 2024 +0200 Merge branch 'main' into ci-bench-cc commit 925cc12 Merge: 3241f93 5889038 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 11:05:42 2024 +0200 Merge branch 'main' into ci-bench-cc commit 3241f93 Merge: 02620a7 d48fbed Author: Lars Eggert <[email protected]> Date: Tue Mar 12 09:59:24 2024 +0200 Merge branch 'main' into ci-bench-cc Signed-off-by: Lars Eggert <[email protected]> commit 02620a7 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 09:57:33 2024 +0200 Try to kill via `$!` commit b32ce9e Merge: 9ea3a99 db1dbb2 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 09:15:18 2024 +0200 Merge branch 'ci-bench-cc' of github.com:larseggert/neqo into ci-bench-cc commit 9ea3a99 Author: Lars Eggert <[email protected]> Date: Tue Mar 12 09:15:05 2024 +0200 Address comments from @martinthomson commit db1dbb2 Merge: 681bbb7 869afea Author: Lars Eggert <[email protected]> Date: Mon Mar 11 19:33:53 2024 +0200 Merge branch 'main' into ci-bench-cc commit 681bbb7 Merge: bd742af 532dcc5 Author: Lars Eggert <[email protected]> Date: Mon Mar 11 18:21:06 2024 +0200 Merge branch 'main' into ci-bench-cc Signed-off-by: Lars Eggert <[email protected]> commit bd742af Author: Lars Eggert <[email protected]> Date: Mon Mar 11 17:00:34 2024 +0200 mkdir -p commit bc7b99f Author: Lars Eggert <[email protected]> Date: Mon Mar 11 16:29:14 2024 +0200 Fix commit e7bf509 Merge: de64b3e cbd4441 Author: Lars Eggert <[email protected]> Date: Mon Mar 11 16:27:56 2024 +0200 Merge branch 'main' into ci-bench-cc commit de64b3e Author: Lars Eggert <[email protected]> Date: Mon Mar 11 16:00:19 2024 +0200 Wait for output before continuing commit 12386a3 Author: Lars Eggert <[email protected]> Date: Mon Mar 11 15:25:40 2024 +0200 ci: Benchmark NewReno and Cubic
mozilla · Mar 16, 2024 · 7f46abb · 7f46abb
1 parent 6bf10f0
commit 7f46abb
Showing 1 changed file with 75 additions and 42 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -9,7 +9,7 @@ env:
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
   RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
-  PERF_CMD: record -o perf.data -F997 --call-graph fp -g
+  PERF_OPT: (record -F997 --call-graph fp -g)
 
 jobs:
   bench:
@@ -73,41 +73,71 @@ jobs:
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
-      # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
-      # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
-      - name: Profile cargo bench transfer
-        run: |
-          # This re-runs part of the previous step, and would hence overwrite part of the criterion results.
-          # Avoid that by shuffling the directories around so this run uses its own results directory.
-          mv target/criterion target/criterion-bench
-          mv target/criterion-transfer-profile target/criterion || true
-          taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
-              --bench --exact "Run multiple transfers with varying seeds" --noplot
-          # And now restore the directories.
-          mv target/criterion target/criterion-transfer-profile
-          mv target/criterion-bench target/criterion
-
-      - name: Profile client/server transfer
+      - name: Compare neqo and msquic
+        env:
+          HOST: 127.0.0.1
+          PORT: 4433
+          SIZE: 134217728 # 128 MB
         run: |
           TMP=$(mktemp -d)
-          { mkdir server; \
-            cd server; \
-            taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
-          mkdir client; \
-            cd client; \
-            taskset -c 1 nice -n -20 time \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-client -- --output-dir "$TMP" "https://$HOST:4433/$SIZE"
-          killall -INT neqo-server
-          cd ${{ github.workspace }}
-          [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
+          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
+          truncate -s "$SIZE" "/tmp/$SIZE"
+          declare -A client_cmd=(
+            ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
+          )
+          declare -A server_cmd=(
+            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
+          )
+
+          function transmogrify {
+            CMD=$1
+            local cc=$2
+            local pacing=$3
+            if [ "$cc" != "" ]; then
+              CMD=${CMD//_cc/--cc $cc}
+              EXT="-$cc"
+            fi
+            if [ "$pacing" == "pacing" ]; then
+              CMD=${CMD//_pacing/--pacing}
+              EXT="$EXT-$pacing"
+            fi
+          }
+
+          for server in neqo msquic; do
+            for client in neqo msquic; do
+              if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
+                cc_opt=("")
+                pacing_opt=("")
+              else
+                cc_opt=("reno" "cubic")
+                pacing_opt=("pacing" "")
+              fi
+              for cc in "${cc_opt[@]}"; do
+                for pacing in "${pacing_opt[@]}"; do
+                  TAG="client $client ← $server server $cc $pacing"
+                  echo "Running benchmarks for $TAG" | tee -a comparison.txt
+                  transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
+                  echo "$CMD"
+                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.server.perf" CMD &
+                  PID=$!
+                  transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
+                  echo "$CMD"
+                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.client.perf" \
+                    hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
+                    tee -a comparison.txt || true
+                  echo >>comparison.txt
+                  kill $PID
+                  {
+                    echo
+                    cat step.md
+                  } >>comparison.md
+                done
+              done
+            done
+          done
           rm -r "$TMP"
-        env:
-          HOST: localhost
-          SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
         env:
@@ -149,20 +179,23 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          perf script -i perf.data -F +pid > transfer.perf &
-          perf script -i client/perf.data -F +pid > client.perf &
-          perf script -i server/perf.data -F +pid > server.perf &
+          perf script -i transfer.perf -F +pid > transfer.fx.perf &
+          for CC in newreno cubic; do
+            for PEER in client server; do
+              perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
+            done
+          done
           wait
-          mv flamegraph.svg transfer.svg
-          mv client/flamegraph.svg client.svg
-          mv server/flamegraph.svg server.svg
           rm neqo.svg
 
       - name: Generate perf reports
         run: |
-          perf report -i perf.data --no-children --stdio > transfer.perf.txt &
-          perf report -i client/perf.data --no-children --stdio > client.perf.txt &
-          perf report -i server/perf.data --no-children --stdio > server.perf.txt &
+          perf report -i transfer.perf --no-children --stdio > transfer.perf.txt &
+          for CC in newreno cubic; do
+            for PEER in client server; do
+              perf report -i "$PEER-$CC.perf" --no-children --stdio > "$PEER-$CC.txt" &
+             done
+          done
           wait
 
       - name: Format results as Markdown