Add PyGLM benchmark subject

Fix incorrect benchmark cases
Refactors of benchmarking and charting

.gitignore

@@ -9,4 +9,6 @@
 /src/spatium/*.pyd
 /src/spatium/_spatium.pyx
 /src/spatium/_spatium.pyi
-/codegen/output/*
+/codegen/output/*
+/benchmark/charts_local/
+/benchmark/results_local/

benchmark/benchmarking.py

@@ -29,7 +29,8 @@
     "numerize",
     "log",
     "temp_log",
-    "indent_log"
+    "indent_log",
+    "CI"
 )
 
 TIMER = time.perf_counter_ns
@@ -225,10 +226,11 @@ def run(self, order_permutations: bool = False, min_runs_per_case: int = 100) ->
             begin = time.perf_counter()
             with temp_log(), indent_log():
                 for i, cases in enumerate(permutations):
-                    with temp_log():
-                        log(f"Sequence[{i+1}/{len(permutations)}]: ", False)
-                        with NoGC:
-                            for _ in range(repeats):
+                    with NoGC:
+                        for rep in range(repeats):
+                            with temp_log():
+                                log(f"Sequence[{(i * repeats + rep + 1)}"
+                                    f"/{len(permutations) * repeats}]: ", False)
                                 for case in cases:
                                     result = case.run()
                                     result.sequence = cases
@@ -328,10 +330,10 @@ def auto_number(self) -> int:
         for self.number in auto_number_series():
             with temp_log():
                 log(numerize(self.number, decimals=1), False)
-                runtime = min(self.run().runtime for _ in range(5))
-                if runtime > AUTO_NUMBER_TARGET_TIME:
-                    return runtime
-                self.number = int(self.number * 1.1)
+                for _ in range(5):
+                    runtime = self.run().runtime
+                    if runtime > AUTO_NUMBER_TARGET_TIME:
+                        return runtime
 
     def run(self) -> "TestCaseResult":
         log(".", False)
@@ -535,6 +537,7 @@ def clear_env():
 
 def save_result(result: BenchmarkResult, file: Path):
     """Save the result and the current environment to a gzipped json file."""
+    file.parent.mkdir(parents=True, exist_ok=True)
     log(f"Saving to {file}...")
     with gzip.open(file, "wt", encoding="utf8") as f:
         json.dump(serialize(result), f, ensure_ascii=False)

benchmark/benchmarks.py

@@ -46,6 +46,12 @@ def numpy():
     a = array((1.0, 2.0, 3.0), dtype=np.float64)
     b = array((3.0, 2.0, 1.0), dtype=np.float64)
 
+@Subject("PyGLM", color="tab:cyan", sort=25)
+def pyglm():
+    from glm import vec3
+    a = vec3(1, 2, 3)
+    b = vec3(3, 2, 1)
+
 
 instantiation = Benchmark("Instantiation")
 @instantiation(spatium, pure_python)
@@ -57,9 +63,12 @@ def pygame():
 @instantiation
 def numpy():
     array((1.0, 2.0, 3.0))
+@instantiation
+def pyglm():
+    vec3(1.0, 2.0, 3.0)
 
 copy = Benchmark("Copy")
-@copy(spatium, pure_python)
+@copy(spatium, pure_python, pyglm)
 def _():
     +a
 @copy(pygame, numpy)
@@ -83,19 +92,28 @@ def _():
 @dot(pygame, numpy)
 def _():
     a.dot(b)
+@dot.setup(pyglm)
+def _():
+    from glm import dot
+@dot
+def pyglm():
+    dot(a, b)
 
 cross = Benchmark("Cross Product")
-@dot(spatium, pure_python)
+@cross(spatium, pure_python)
 def _():
     a ^ b
-@dot
+@cross
 def pygame():
     a.cross(b)
-@dot.setup(numpy)
+@cross.setup(numpy)
 def _():
     cross = np.cross
-@dot
-def numpy():
+@cross.setup(pyglm)
+def _():
+    from glm import cross
+@cross(numpy, pyglm)
+def _():
     cross(a, b)
 
 equality = Benchmark("Equality")
@@ -115,6 +133,12 @@ def _():
 @length
 def pygame():
     a.length()
+@length.setup(pyglm)
+def _():
+    from glm import length
+@length
+def pyglm():
+    length(a)
 
 normalize = Benchmark("Normalize")
 @normalize(spatium, pure_python)
@@ -129,6 +153,12 @@ def _():
 @normalize
 def numpy():
     norm(a)
+@normalize.setup(pyglm)
+def _():
+    from glm import normalize
+@normalize
+def pyglm():
+    normalize(a)
 
 get_item = Benchmark("Get Item")
 @get_item
@@ -141,18 +171,18 @@ def _all_():
     a[1] = 4.0
 
 swizzle_get = Benchmark("Swizzle Get")
-@swizzle_get(spatium, pure_python, pygame)
+@swizzle_get(spatium, pure_python, pygame, pyglm)
 def _():
     a.zxy
 
 swizzle_set = Benchmark("Swizzle Set")
-@swizzle_set(spatium, pure_python, pygame)
+@swizzle_set(spatium, pure_python, pygame, pyglm)
 def _():
-    a.zxy
+    a.zxy = b
 
 
 result = run_benchmarks(
     order_permutations=True,
     min_runs_per_case=100
 )
-save_result(result, Path("results", result.datetime.strftime("%Y%m%d_%H-%M-%S") + ".dat"))
+save_result(result, Path("results" if CI else "results_local", result.datetime.strftime("%Y%m%d_%H-%M-%S") + ".dat"))

benchmark/gen_charts.py

@@ -3,17 +3,20 @@
 import shutil
 
 import charting
-from benchmarking import load_result, clear_env, Path, Subject, log, indent_log
+from benchmarking import load_result, clear_env, Path, Subject, log, indent_log, CI
 
+RESULTS = Path("results" if CI else "results_local")
+CHARTS = Path("charts" if CI else "charts_local")
+CHARTS.mkdir(parents=True, exist_ok=True)
 
-files = [f[:-4] for f in os.listdir("results") if f.endswith(".dat")]
+files = [Path(f) for f in os.listdir(RESULTS) if f.endswith(".dat")]
 # Latest to earliest
-files.sort(reverse=True, key=lambda f: datetime.datetime.strptime(f, "%Y%m%d_%H-%M-%S"))
+files.sort(reverse=True, key=lambda f: datetime.datetime.strptime(f.stem, "%Y%m%d_%H-%M-%S"))
 
 for file in files:
     log(f"{file}:")
     with indent_log():
-        result = load_result(Path(f"results/{file}.dat"))
+        result = load_result(RESULTS / file)
         log("Generating chart...")
         chart = charting.chart(
             result,
@@ -28,18 +31,18 @@
                 ,
             )
         )
-        chart.savefig(f"charts/{file}.svg")
+        chart.savefig(CHARTS / file.with_suffix(".svg"))
         clear_env()
 
 log()
 log(f"Latest: {files[0]}")
-shutil.copyfile(f"charts/{files[0]}.svg", f"charts/latest.svg")
+shutil.copyfile(CHARTS / files[0].with_suffix(".svg"), CHARTS / "latest.svg")
 
 log()
 log("Generating README.md")
-with open("charts/README.md", "w") as f:
+with open(CHARTS / "README.md", "w") as f:
     f.write("# All Benchmarks (Reverse Chronological Order)\n")
     f.write("---\n")
     f.write("\n")
     for file in files:
-        f.write(f"[![{file}](./{file}.svg)](./{file}.svg)\n")
+        f.write(f"[![{file.stem}](./{file.with_suffix(".svg")})](./{file.with_suffix(".svg")})\n")

benchmark/requirements.txt

@@ -4,4 +4,5 @@ matplotlib
 numerize
 psutil
 py-cpuinfo
-colorama
+colorama
+PyGLM