Merge pull request #481 from dyson-ai/feature/name_cache_config_params

rename use_cache and use_sample_cache to cache_results and cache_samples

bencher/bench_cfg.py

@@ -83,16 +83,16 @@ class BenchRunCfg(BenchPlotSrvCfg):
 
     raise_duplicate_exception: bool = param.Boolean(False, doc=" Used to debug unique plot names.")
 
-    use_cache: bool = param.Boolean(
+    cache_results: bool = param.Boolean(
         False,
-        doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete.  If you want to cache values during the benchmark you need to use the use_sample_cache option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
+        doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete.  If you want to cache values during the benchmark you need to use the cache_samples option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
     )
 
     clear_cache: bool = param.Boolean(
         False, doc=" Clear the cache of saved input->output mappings."
     )
 
-    use_sample_cache: bool = param.Boolean(
+    cache_samples: bool = param.Boolean(
         False,
         doc="If true, every time the benchmark function is called, bencher will check if that value has been calculated before and if so load the from the cache.  Note that the sample level cache is different from the benchmark level cache which only caches the aggregate of all the results at the end of the benchmark. This cache lets you stop a benchmark halfway through and continue. However, beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
     )
@@ -182,7 +182,7 @@ def from_cmd_line() -> BenchRunCfg:  # pragma: no cover
         parser.add_argument(
             "--use-cache",
             action="store_true",
-            help=BenchRunCfg.param.use_cache.doc,
+            help=BenchRunCfg.param.cache_results.doc,
         )
 
         parser.add_argument(
@@ -380,8 +380,8 @@ def describe_benchmark(self) -> str:
             benchmark_sampling_str.append(f"    run tag: {self.run_tag}")
         if self.level is not None:
             benchmark_sampling_str.append(f"    bench level: {self.level}")
-        benchmark_sampling_str.append(f"    use_cache: {self.use_cache}")
-        benchmark_sampling_str.append(f"    use_sample_cache: {self.use_sample_cache}")
+        benchmark_sampling_str.append(f"    cache_results: {self.cache_results}")
+        benchmark_sampling_str.append(f"    cache_samples {self.cache_samples}")
         benchmark_sampling_str.append(f"    only_hash_tag: {self.only_hash_tag}")
         benchmark_sampling_str.append(f"    executor: {self.executor}")
 

bencher/bench_runner.py

@@ -33,11 +33,11 @@ def __init__(
 
     @staticmethod
     def setup_run_cfg(
-        run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, use_cache=True
+        run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, cache_results=True
     ) -> BenchRunCfg:
         run_cfg_out = deepcopy(run_cfg)
-        run_cfg_out.use_sample_cache = use_cache
-        run_cfg_out.only_hash_tag = use_cache
+        run_cfg_out.cache_samples = cache_results
+        run_cfg_out.only_hash_tag = cache_results
         run_cfg_out.level = level
         return run_cfg_out
 
@@ -78,9 +78,9 @@ def run(
         show: bool = False,
         save: bool = False,
         grouped: bool = True,
-        use_cache: bool = True,
+        cache_results: bool = True,
     ) -> List[Bench]:
-        """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow.  If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default use_cache=True so that previous values are reused.
+        """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow.  If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default cache_results=True so that previous values are reused.
 
         Args:
             min_level (int, optional): The minimum level to start sampling at. Defaults to 2.
@@ -93,14 +93,14 @@ def run(
             show (bool, optional): show the results in the local web browser. Defaults to False.
             save (bool, optional): save the results to disk in index.html. Defaults to False.
             grouped (bool, optional): Produce a single html page with all the benchmarks included. Defaults to True.
-            use_cache (bool, optional): Use the sample cache to reused previous results. Defaults to True.
+            cache_results (bool, optional): Use the sample cache to reused previous results. Defaults to True.
 
         Returns:
             List[BenchCfg]: A list of bencher instances
         """
         if run_cfg is None:
             run_cfg = deepcopy(self.run_cfg)
-        run_cfg = BenchRunner.setup_run_cfg(run_cfg, use_cache=use_cache)
+        run_cfg = BenchRunner.setup_run_cfg(run_cfg, cache_results=cache_results)
 
         if level is not None:
             min_level = level

bencher/bencher.py

@@ -327,7 +327,7 @@ def plot_sweep(
                 logging.info("Copy run cfg from bench class")
 
         if run_cfg.only_plot:
-            run_cfg.use_cache = True
+            run_cfg.cache_results = True
 
         self.last_run_cfg = run_cfg
 
@@ -448,7 +448,7 @@ def run_sweep(
             if run_cfg.clear_cache:
                 c.delete(bench_cfg_hash)
                 logging.info("cleared cache")
-            elif run_cfg.use_cache:
+            elif run_cfg.cache_results:
                 logging.info(
                     f"checking for previously calculated results with key: {bench_cfg_hash}"
                 )
@@ -813,7 +813,7 @@ def init_sample_cache(self, run_cfg: BenchRunCfg):
             cache_name="sample_cache",
             tag_index=True,
             size_limit=self.cache_size,
-            use_cache=run_cfg.use_sample_cache,
+            cache_results=run_cfg.cache_samples,
         )
 
     def clear_tag_from_sample_cache(self, tag: str, run_cfg):

bencher/example/example_composable_container.py

@@ -98,7 +98,7 @@ def example_composable_container_video(
 
 if __name__ == "__main__":
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = False
+    ex_run_cfg.cache_samples = False
     # ex_run_cfg.level = 2
     ex_report = bch.BenchReport()
     example_composable_container_image(ex_run_cfg, report=ex_report)

bencher/example/example_composable_container2.py

@@ -144,7 +144,7 @@ def example_composable_container_image(
 
 # if __name__ == "__main__":
 #     ex_run_cfg = bch.BenchRunCfg()
-#     ex_run_cfg.use_sample_cache = False
+#     ex_run_cfg.cache_samples = False
 #     # ex_run_cfg.level = 2
 #     ex_report = bch.BenchReport()
 #     example_composable_container_image(ex_run_cfg, report=ex_report)
@@ -157,4 +157,4 @@ def example_composable_container_image(
     # bench_runner.add_run(bench_image)
     bench_runner.add_run(example_composable_container_image)
 
-    bench_runner.run(level=6, show=True, use_cache=False)
+    bench_runner.run(level=6, show=True, cache_results=False)

bencher/example/example_holosweep.py

@@ -95,4 +95,4 @@ def example_holosweep(
     PlotFunctions().to_gui()
     bench_run = bch.BenchRunner("bench_runner_test")
     bench_run.add_run(example_holosweep)
-    bench_run.run(level=6, show=True, use_cache=False)
+    bench_run.run(level=6, show=True, cache_results=False)

bencher/example/example_image.py

@@ -58,7 +58,7 @@ def points_to_polygon_png(self, points: list[float], filename: str):
 def example_image(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
-    run_cfg.use_cache = False
+    run_cfg.cache_results = False
     bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
 
     bench.result_vars = ["polygon", "area"]
@@ -142,7 +142,7 @@ def example_image_vid_sequential(
     # def example_image_pairs()
 
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     # ex_run_cfg.debug = True
     # ex_run_cfg.repeats = 2
     ex_run_cfg.level = 4

bencher/example/example_image1.py

@@ -73,7 +73,7 @@ def example_image_vid_sequential1(
 
 if __name__ == "__main__":
     ex_run_cfg = bch.BenchRunCfg()
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     ex_run_cfg.overwrite_sample_cache = True
     ex_run_cfg.level = 3
 

bencher/example/example_sample_cache.py

@@ -2,7 +2,7 @@
 
 
 class UnreliableClass(bch.ParametrizedSweep):
-    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.use_sample_cache you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
+    """This class helps demonstrate benchmarking a function that sometimes crashes during sampling.  By using BenchRunCfg.cache_samples you can store the results of every call to the benchmark function so data is not lost in the event of a crash.  However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data.  For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
 
     input_val = bch.IntSweep(
         default=0,
@@ -31,7 +31,7 @@ def example_sample_cache(
     report: bch.BenchReport = bch.BenchReport(),
     trigger_crash: bool = False,
 ) -> bch.Bench:
-    """This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
+    """This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
 
     Args:
         run_cfg (BenchRunCfg): configuration of how to perform the param sweep
@@ -50,7 +50,7 @@ def example_sample_cache(
         title="Example Crashy Function with the sample_cache",
         input_vars=[UnreliableClass.param.input_val],
         result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash],
-        description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
+        description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
         run_cfg=run_cfg,
         post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
     )
@@ -63,7 +63,7 @@ def example_sample_cache(
     ex_run_cfg.executor = bch.Executors.SCOOP
 
     # this will store the result of of every call to crashy_fn
-    ex_run_cfg.use_sample_cache = True
+    ex_run_cfg.cache_samples = True
     ex_run_cfg.clear_sample_cache = True
 
     try:

bencher/example/example_sample_cache_context.py

@@ -51,7 +51,7 @@ def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_ca
 
 def example_cache_context() -> bch.Bench:
     run_cfg = bch.BenchRunCfg()
-    run_cfg.use_sample_cache = True
+    run_cfg.cache_samples = True
     run_cfg.only_hash_tag = True
     run_cfg.repeats = 2
     run_cfg.parallel = False

bencher/example/example_video.py

@@ -79,7 +79,7 @@ def example_video(
     run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
 ) -> bch.Bench:
     # run_cfg.auto_plot = False
-    # run_cfg.use_sample_cache = True
+    # run_cfg.cache_samples = True
     bench = bch.Bench("example_video", TuringPattern(), run_cfg=run_cfg, report=report)
 
     bench.plot_sweep(
@@ -111,7 +111,7 @@ def example_video_tap(
 if __name__ == "__main__":
     run_cfg_ex = bch.BenchRunCfg()
     run_cfg_ex.level = 2
-    run_cfg_ex.use_sample_cache = True
+    run_cfg_ex.cache_samples = True
     run_cfg_ex.only_hash_tag = True
 
     # example_video(run_cfg_ex).report.show()

bencher/example/experimental/example_hvplot_explorer.py

@@ -30,7 +30,7 @@ def bench_function(cfg: ExampleBenchCfgIn) -> ExampleBenchCfgOut:
         post_description="Here you can see the output plot of sin theta between 0 and pi.  In the tabs at the top you can also view 3 tabular representations of the data",
         run_cfg=bch.BenchRunCfg(
             auto_plot=True,
-            use_cache=False,
+            cache_results=False,
             repeats=2,
         ),
     )

bencher/example/shelved/example_float3D_cone.py

@@ -92,5 +92,5 @@
 
 # if __name__ == "__main__":
 #     ex_run_cfg = bch.BenchRunCfg()
-#     ex_run_cfg.use_cache = True
+#     ex_run_cfg.cache_results = True
 #     example_cone(ex_run_cfg).report.show()

bencher/job.py

@@ -78,11 +78,11 @@ def __init__(
         cache_name: str = "fcache",
         tag_index: bool = True,
         size_limit: int = int(20e9),  # 20 GB
-        use_cache=True,
+        cache_results=True,
     ):
         self.executor_type = executor
         self.executor = None
-        if use_cache:
+        if cache_results:
             self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit)
             logging.info(f"cache dir: {self.cache.directory}")
         else:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "holobench"
-version = "1.35.0"
+version = "1.36.0"
 
 authors = [{ name = "Austin Gregg-Smith", email = "[email protected]" }]
 description = "A package for benchmarking the performance of arbitrary functions"

test/test_bench_runner.py

@@ -9,7 +9,7 @@ class TestBenchRunner(unittest.TestCase):
     # Tests that bch.BenchRunner can be created with default configuration and the import statement in the bch.BenchRunner class is fixed
     def test_benchrunner_default_configuration_fixed(self):
         bench_runner = bch.BenchRunner("bench_runner_test")
-        self.assertEqual(bench_runner.run_cfg.use_sample_cache, True)
+        self.assertEqual(bench_runner.run_cfg.cache_samples, True)
         self.assertEqual(bench_runner.run_cfg.only_hash_tag, True)
         self.assertEqual(bench_runner.run_cfg.level, 2)
         self.assertEqual(bench_runner.publisher, None)
@@ -71,8 +71,8 @@ def run_bench_class(run_cfg: bch.BenchRunCfg, report: bch.BenchReport) -> bch.Be
         self.assertEqual(results[0].sample_cache.worker_cache_call_count, 0)
         self.assertEqual(results[0].run_cfg.run_tag, run_tag)
 
-        # run with the same tag but set use cache to false, should not hit cache because even tho the tag is the same, use_cache=false
-        results = bench_runner.run(use_cache=False)
+        # run with the same tag but set use cache to false, should not hit cache because even tho the tag is the same, cache_results=false
+        results = bench_runner.run(cache_results=False)
         self.assertEqual(results[0].sample_cache.worker_wrapper_call_count, 2)
         self.assertEqual(results[0].sample_cache.worker_fn_call_count, 2)
         self.assertEqual(results[0].sample_cache.worker_cache_call_count, 0)
@@ -144,7 +144,7 @@ def test_benchrunner_repeats(self):
     #     bench_runner.add_run(bench_fn1)
     #     bench_runner.add_run(bench_fn2)
     #     run_cfg = bch.BenchRunCfg()
-    #     run_cfg.use_sample_cache = False
+    #     run_cfg.cache_samples = False
     #     run_cfg.only_hash_tag = False
     #     run_cfg.level = 3
     #     results = bench_runner.run(run_cfg=run_cfg)

test/test_bencher.py

@@ -293,7 +293,7 @@ def test_benching_cache_without_time(self, over_time) -> None:
             title=title,
             input_vars=iv,
             result_vars=rv,
-            run_cfg=BenchRunCfg(over_time=over_time, use_cache=False, auto_plot=False),
+            run_cfg=BenchRunCfg(over_time=over_time, cache_results=False, auto_plot=False),
         )
         self.assertEqual(
             bench2.sample_cache.worker_wrapper_call_count, ExampleBenchCfgIn.param.theta.samples
@@ -305,7 +305,7 @@ def test_benching_cache_without_time(self, over_time) -> None:
             title=title,
             input_vars=iv,
             result_vars=rv,
-            run_cfg=BenchRunCfg(over_time=over_time, use_cache=True, auto_plot=False),
+            run_cfg=BenchRunCfg(over_time=over_time, cache_results=True, auto_plot=False),
         )
         self.assertEqual(
             bench2.sample_cache.worker_wrapper_call_count, ExampleBenchCfgIn.param.theta.samples
@@ -348,7 +348,7 @@ def test_const_hashing(self, noisy) -> None:
             const_vars=[
                 (ExampleBenchCfgIn.param.noisy, noisy),
             ],
-            run_cfg=BenchRunCfg(use_cache=True, auto_plot=False),
+            run_cfg=BenchRunCfg(cache_results=True, auto_plot=False),
         )
         # the result should be cached so the call count should be the same as before
         self.assertEqual(

test/test_sample_cache.py

@@ -13,7 +13,7 @@ def call_bencher(self, bencher, run_cfg):
                 UnreliableClass.param.return_value,
                 UnreliableClass.param.trigger_crash,
             ],
-            description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashing during the run""",
+            description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashing during the run""",
             run_cfg=run_cfg,
             post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
             tag="testing_tag3",
@@ -29,7 +29,7 @@ def sample_cache(self):
         run_cfg.repeats = 1
         run_cfg.executor = bch.Executors.SERIAL  # THE ASSERTS WILL ONLY WORK IF RUN SERIALLY!!!
 
-        run_cfg.use_sample_cache = True  # this will store the result of every call
+        run_cfg.cache_samples = True  # this will store the result of every call
         run_cfg.only_hash_tag = True
         run_cfg.auto_plot = False