From 2227fdd91eb8408062e59cca36140bbb5e38e27a Mon Sep 17 00:00:00 2001 From: Brendan Chou <3680392+BrendanChou@users.noreply.github.com> Date: Thu, 16 May 2024 13:42:18 -0700 Subject: [PATCH 1/5] add a protocol benchmark run --- .github/workflows/protocol-benchmark.yml | 38 ++++++++++++++++++++++++ protocol/Makefile | 4 +-- 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/protocol-benchmark.yml diff --git a/.github/workflows/protocol-benchmark.yml b/.github/workflows/protocol-benchmark.yml new file mode 100644 index 0000000000..cdd50bbcea --- /dev/null +++ b/.github/workflows/protocol-benchmark.yml @@ -0,0 +1,38 @@ +name: Protocol Benchmark +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - 'protocol/**' + push: + branches: + - main + - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x + - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x + paths: + - 'protocol/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + benchmark: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./protocol + steps: + - + name: Check out repository code + uses: actions/checkout@v3 + - + name: Setup Golang + uses: actions/setup-go@v3 + with: + go-version: 1.22 + - + name: Ensure `go.mod` is up to date + run: go mod tidy && git diff --exit-code + - + name: Run Benchmarks + run: make benchmark diff --git a/protocol/Makefile b/protocol/Makefile index d21eb55330..cfb05c8efd 100644 --- a/protocol/Makefile +++ b/protocol/Makefile @@ -231,7 +231,7 @@ test-unit-and-integration: @VERSION=$(VERSION) go test -mod=readonly -tags='integration_test test_ledger_mock $(build_tags)' ./... benchmark: - @VERSION=$(VERSION) go test -mod=readonly -tags='$(build_tags)' -bench=. ./... + @VERSION=$(VERSION) go test -mod=readonly -tags='$(build_tags)' -run=^# -bench=. -count 5 -benchmem ./... test-container: @SKIP_DISABLED=true VERSION=$(VERSION) go test -mod=readonly -tags='container_test $(build_tags)' ./... @@ -421,6 +421,6 @@ update-sample-pregenesis: check-sample-pregenesis-up-to-date: $(MAKE) localnet-build @docker build . -t check-sample-pregenesis -f scripts/genesis/Dockerfile --no-cache - @docker run -v $(CURDIR):/workspace check-sample-pregenesis + @docker run -v $(CURDIR):/workspace check-sample-pregenesis .PHONY: update-sample-pregenesis check-sample-pregenesis-up-to-date From f75f7896731ae431a8e5ef582632e19a65731323 Mon Sep 17 00:00:00 2001 From: Brendan Chou <3680392+BrendanChou@users.noreply.github.com> Date: Thu, 16 May 2024 13:55:31 -0700 Subject: [PATCH 2/5] add a benchmark --- protocol/lib/big_math_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/protocol/lib/big_math_test.go b/protocol/lib/big_math_test.go index 4d83dd7d33..e4e077fc00 100644 --- a/protocol/lib/big_math_test.go +++ b/protocol/lib/big_math_test.go @@ -387,6 +387,16 @@ func TestBigRatMulPpm(t *testing.T) { } } +func BenchmarkBigRatMulPpm(b *testing.B) { + input := big.NewRat(1_000_000, 1_234_567) + ppm := uint32(5) + var result *big.Rat + for i := 0; i < b.N; i++ { + result = lib.BigRatMulPpm(input, ppm) + } + require.Equal(b, big.NewRat(5, 1_234_567), result) +} + func TestBigRatClamp(t *testing.T) { tests := map[string]struct { input *big.Rat From bcfd6e570b8c0c087f8688e36c537aaac4e7a42d Mon Sep 17 00:00:00 2001 From: Brendan Chou <3680392+BrendanChou@users.noreply.github.com> Date: Thu, 16 May 2024 14:31:05 -0700 Subject: [PATCH 3/5] improve benchmark action --- .github/workflows/protocol-benchmark.yml | 33 +++++++++++++++++------- protocol/Makefile | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/protocol-benchmark.yml b/.github/workflows/protocol-benchmark.yml index cdd50bbcea..f615931998 100644 --- a/.github/workflows/protocol-benchmark.yml +++ b/.github/workflows/protocol-benchmark.yml @@ -22,17 +22,30 @@ jobs: run: working-directory: ./protocol steps: - - - name: Check out repository code + - name: Check out repository code uses: actions/checkout@v3 - - - name: Setup Golang + - name: Setup Golang uses: actions/setup-go@v3 with: go-version: 1.22 - - - name: Ensure `go.mod` is up to date - run: go mod tidy && git diff --exit-code - - - name: Run Benchmarks - run: make benchmark + - name: Run Benchmarks + run: make benchmark | tee benchmark_output.txt + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: benchmark_output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: benchmark_output.txt + external-data-json-path: ./cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + alert-comment-cc-users: '@brendanchou' diff --git a/protocol/Makefile b/protocol/Makefile index cfb05c8efd..064d101c7d 100644 --- a/protocol/Makefile +++ b/protocol/Makefile @@ -231,7 +231,7 @@ test-unit-and-integration: @VERSION=$(VERSION) go test -mod=readonly -tags='integration_test test_ledger_mock $(build_tags)' ./... benchmark: - @VERSION=$(VERSION) go test -mod=readonly -tags='$(build_tags)' -run=^# -bench=. -count 5 -benchmem ./... + @VERSION=$(VERSION) go test -mod=readonly -tags='$(build_tags)' -run=^# -bench=. -benchtime=5s -benchmem ./... test-container: @SKIP_DISABLED=true VERSION=$(VERSION) go test -mod=readonly -tags='container_test $(build_tags)' ./... From b086d3273ceae7029f602e7e5eeb5d6c79119c36 Mon Sep 17 00:00:00 2001 From: Brendan Chou <3680392+BrendanChou@users.noreply.github.com> Date: Thu, 16 May 2024 14:51:22 -0700 Subject: [PATCH 4/5] update --- .github/workflows/protocol-benchmark.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/protocol-benchmark.yml b/.github/workflows/protocol-benchmark.yml index f615931998..1f03a5dba4 100644 --- a/.github/workflows/protocol-benchmark.yml +++ b/.github/workflows/protocol-benchmark.yml @@ -46,6 +46,5 @@ jobs: tool: 'go' output-file-path: benchmark_output.txt external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - alert-comment-cc-users: '@brendanchou' + fail-on-alert: true + alert-threshold: '150%' From 1d468384c9e3b857b2432d822f837acb71cd955e Mon Sep 17 00:00:00 2001 From: Brendan Chou <3680392+BrendanChou@users.noreply.github.com> Date: Thu, 16 May 2024 16:04:37 -0700 Subject: [PATCH 5/5] try update benchmark workflow --- .github/workflows/protocol-benchmark.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/protocol-benchmark.yml b/.github/workflows/protocol-benchmark.yml index 1f03a5dba4..8b7d193f05 100644 --- a/.github/workflows/protocol-benchmark.yml +++ b/.github/workflows/protocol-benchmark.yml @@ -2,14 +2,14 @@ name: Protocol Benchmark on: # yamllint disable-line rule:truthy pull_request: paths: - - 'protocol/**' + - 'protocol/lib/**' push: branches: - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x paths: - - 'protocol/**' + - 'protocol/lib/**' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -29,12 +29,7 @@ jobs: with: go-version: 1.22 - name: Run Benchmarks - run: make benchmark | tee benchmark_output.txt - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'go' - output-file-path: benchmark_output.txt + run: make benchmark | tee ./benchmark_output.txt - name: Download previous benchmark data uses: actions/cache@v4 with: @@ -44,7 +39,8 @@ jobs: uses: benchmark-action/github-action-benchmark@v1 with: tool: 'go' - output-file-path: benchmark_output.txt + output-file-path: ./protocol/benchmark_output.txt external-data-json-path: ./cache/benchmark-data.json fail-on-alert: true alert-threshold: '150%' + save-data-file: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}