cicd: benchmarking via GH actions (#21)

cicd: benchmarking via GH actions

Benchmark the current HEAD as well as the master branch and compare
performance. Store the results as an artifact into the action.

The repository does not allow commenting on issues using the Github token so we cannot comment the outcome on the PR this might be tackled in the future.

.github/workflows/bench.yml

@@ -0,0 +1,62 @@
+# As inspired by https://dev.to/vearutop/continuous-benchmarking-with-go-and-github-actions-41ok
+name: bench
+on:
+  pull_request:
+env:
+  GO111MODULE: "on"
+jobs:
+  bench:
+    permissions:
+        pull-requests: write
+        issues: write 
+    strategy:
+      matrix:
+        go-version: [ 1.22.x ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          path: ~/go/pkg
+          key: ${{ runner.os }}-go-pkg-${{ hashFiles('**/go.mod') }}
+      - uses: actions/cache@v4
+        with:
+          path: testing/venv
+          key: ${{ runner.os }}-py-venv-${{ hashFiles('**/requirements.txt') }}
+      - uses: actions/cache@v4
+        with:
+          path: cmd/bench-main.txt
+          key: ${{ runner.os }}-bench10-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
+      - name: Benchmark
+        run: |
+          make setup-test-dependencies
+          make start-test-s3-servers
+          make bench-main
+          REF_NAME=${GITHUB_REF##*/} make bench-current
+          echo "\n\n\n"
+          SCRIPT_OUTPUT="$(make bench-report)"
+          echo "SCRIPT_OUTPUT<<EOF" >> $GITHUB_ENV
+          echo "$SCRIPT_OUTPUT" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+    #   Requires enablement at organization level
+    #   - name: Add benchmark report to PR
+    #     uses: actions/github-script@v7
+    #     with:
+    #         github-token: ${{secrets.GITHUB_TOKEN}}
+    #         script: |
+    #             github.rest.issues.createComment({
+    #                 issue_number: context.issue.number,
+    #                 owner: context.repo.owner,
+    #                 repo: context.repo.repo,
+    #                 body: `${{env.SCRIPT_OUTPUT}}`
+    #             })
+      - uses: actions/upload-artifact@v4
+        with:
+            name: benchmark-report-pr-${{ github.event.number }}
+            path: cmd/bench-*.txt
+            overwrite: true

.gitignore

@@ -5,4 +5,6 @@
 
 testing/venv/**
 testing/**.err
-testing/**.log
+testing/**.log
+
+cmd/bench-*.txt

Makefile

@@ -1,3 +1,6 @@
+BENCH_COUNT ?= 10
+
+
 build-container:
 	podman build -t fakes3pp -f Dockerfile .
 
@@ -19,3 +22,22 @@ start-test-s3-servers:
 
 stop-test-s3-servers:
 	for pid in `ps -ef | grep testing/venv/moto/bin/python3 | grep -v grep | awk '{print $$2}'`; do kill "$${pid}"; done
+
+bench-dependencies:
+	@test -s $(GOPATH)/bin/benchstat || go install golang.org/x/perf/cmd/benchstat@latest
+
+bench-main: bench-dependencies
+	git remote -v
+	git fetch origin
+	git rev-parse --short HEAD 2>/dev/null | tr -d '\n' | tee cmd/bench-current_ref.txt
+	git branch -d "before_going_to_main" || echo "If there was no branch before_going_to_main then this is ok"
+	git checkout -b "before_going_to_main"
+	git checkout origin/main
+	test -e cmd/bench-main.txt || (cd cmd && go test -bench=. -benchtime=5s -run "FakeS3Proxy" -benchmem -count=$(BENCH_COUNT) | tee bench-main.txt && cd ..)
+	git checkout "before_going_to_main"
+
+bench-current: bench-dependencies
+	cd cmd && go test -bench=. -benchtime=5s -run "FakeS3Proxy" -benchmem -count=$(BENCH_COUNT) | tee bench-$(shell git rev-parse --short HEAD 2>/dev/null).txt && cd ..
+
+bench-report: bench-dependencies
+	benchstat cmd/bench-main.txt cmd/bench-$(shell git rev-parse --short HEAD 2>/dev/null).txt | tee cmd/bench-$(shell git rev-parse --short HEAD 2>/dev/null)-master-report.txt