Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make tests (excluding RegressionTests.jl) run 37% faster #141

Merged
merged 3 commits into from
Nov 24, 2024
Merged

Make tests (excluding RegressionTests.jl) run 37% faster #141

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cb09901
full pipeline seconds=.001 (29.3=>22.9)
LilithHafner Nov 24, 2024
735f162
skip Aqua's persistent_task tests (22.9s => 20.5s)
LilithHafner Nov 24, 2024
c679f97
assorted other changes (reducing runtime budgets) (20.5s => 18.3s)
LilithHafner Nov 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ using Random: rand!

@testset "passing value into setup" begin
x = rand(100)
@b sort(x)
@b x sort # This was previously broken
@b rand hash
@b sort(x) seconds=.01
@b x sort seconds=.01# This was previously broken
@b rand hash seconds=.01
x = rand
@b x hash
@b x hash seconds=.01
end

@testset "blank space" begin
@test (@b sleep(.01) identity).time < .01 < (@b _ sleep(.01) identity).time
@test (@b sleep(.005) identity seconds=.01).time < .005 < (@b _ sleep(.005) identity seconds=.01).time
end

@testset "Median" begin
Expand Down Expand Up @@ -158,31 +158,32 @@ using Random: rand!
@test Chairmarks.DEFAULTS.gc === true
Chairmarks.DEFAULTS.seconds = 1
@test Chairmarks.DEFAULTS.seconds === 1.0
@test 1 <= @elapsed @b 1+1
Chairmarks.DEFAULTS.seconds = 0.3
@test 0.3 <= @elapsed @b 1+1
Chairmarks.DEFAULTS.seconds = 0.1
@test Chairmarks.DEFAULTS.seconds === 0.1
end

@testset "Comparative benchmarking" begin
# Basic
x,y = @b .001 sleep,sleep(10*_)
x,y = @b .0001 sleep,sleep(10*_) seconds=.01
@test x.evals == y.evals
@test x.warmup == y.warmup
x,y = @be .001 sleep,sleep(10*_)
x,y = @be .0001 sleep,sleep(10*_) seconds=.01
@test length(x.samples) == length(y.samples)

# Low sample count
@b rand,hash(rand()) samples=0 evals=1
@b rand,hash(rand()) seconds=0
@b rand,sleep(.2)
@b rand,sleep(.02) seconds=.01

# Full pipeline and order of evals
log = []
_push!(x, v) = (push!(x, v); v)
x,y = @be _push!(log, (0,)) _push!(log, (_...,1)) _push!(log, (_...,2)), _push!(log, (_...,3)) _push!(log, (_...,4))
x,y = @be _push!(log, (0,)) _push!(log, (_...,1)) _push!(log, (_...,2)), _push!(log, (_...,3)) _push!(log, (_...,4)) seconds=.001

# Sanity
all(∈(((0,), (0,1), (0,1,2), (0,1,3), (0,1,2,4), (0,1,3,4))), log)
all(∈([(0,), (0,1), (0,1,2), (0,1,3), (0,1,2,4), (0,1,3,4)]), log)
evals = Chairmarks.only(unique(x.evals for x in x.samples))
@test Chairmarks.only(unique(y.evals for y in y.samples)) == evals

Expand Down Expand Up @@ -247,7 +248,7 @@ using Random: rand!
glog = UInt[]
f(x) = (@assert !issorted(x); push!(flog, hash(x)); sort!(x; alg=QuickSort))
g(x) = (@assert !issorted(x); push!(glog, hash(x)); sort!(x; alg=InsertionSort))
@b rand!(a),copyto!(b,a) f(_[1]),g(_[2]) (@assert issorted(_::Vector{Int})) evals=1
@b rand!(a),copyto!(b,a) f(_[1]),g(_[2]) (@assert issorted(_::Vector{Int})) evals=1 seconds=.01
@test flog == glog

# More than two test functions
Expand Down Expand Up @@ -519,7 +520,8 @@ using Random: rand!

@testset "Aqua" begin
import Aqua
Aqua.test_all(Chairmarks, deps_compat=false)
# persistent_tasks=false because that test is slow and we don't use persistent tasks
Aqua.test_all(Chairmarks, deps_compat=false, persistent_tasks=false)
Aqua.test_deps_compat(Chairmarks, check_extras=false)
end

Expand Down
Loading