diff --git a/bencher/bench_cfg.py b/bencher/bench_cfg.py index 9923576b..20cbd446 100644 --- a/bencher/bench_cfg.py +++ b/bencher/bench_cfg.py @@ -83,16 +83,16 @@ class BenchRunCfg(BenchPlotSrvCfg): raise_duplicate_exception: bool = param.Boolean(False, doc=" Used to debug unique plot names.") - use_cache: bool = param.Boolean( + cache_results: bool = param.Boolean( False, - doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete. If you want to cache values during the benchmark you need to use the use_sample_cache option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.", + doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete. If you want to cache values during the benchmark you need to use the cache_samples option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.", ) clear_cache: bool = param.Boolean( False, doc=" Clear the cache of saved input->output mappings." ) - use_sample_cache: bool = param.Boolean( + cache_samples: bool = param.Boolean( False, doc="If true, every time the benchmark function is called, bencher will check if that value has been calculated before and if so load the from the cache. Note that the sample level cache is different from the benchmark level cache which only caches the aggregate of all the results at the end of the benchmark. This cache lets you stop a benchmark halfway through and continue. However, beware that depending on how you change code in the objective function, the cache could provide values that are not correct.", ) @@ -182,7 +182,7 @@ def from_cmd_line() -> BenchRunCfg: # pragma: no cover parser.add_argument( "--use-cache", action="store_true", - help=BenchRunCfg.param.use_cache.doc, + help=BenchRunCfg.param.cache_results.doc, ) parser.add_argument( @@ -380,8 +380,8 @@ def describe_benchmark(self) -> str: benchmark_sampling_str.append(f" run tag: {self.run_tag}") if self.level is not None: benchmark_sampling_str.append(f" bench level: {self.level}") - benchmark_sampling_str.append(f" use_cache: {self.use_cache}") - benchmark_sampling_str.append(f" use_sample_cache: {self.use_sample_cache}") + benchmark_sampling_str.append(f" cache_results: {self.cache_results}") + benchmark_sampling_str.append(f" cache_samples {self.cache_samples}") benchmark_sampling_str.append(f" only_hash_tag: {self.only_hash_tag}") benchmark_sampling_str.append(f" executor: {self.executor}") diff --git a/bencher/bench_runner.py b/bencher/bench_runner.py index 1ebfb123..3698e10c 100644 --- a/bencher/bench_runner.py +++ b/bencher/bench_runner.py @@ -33,11 +33,11 @@ def __init__( @staticmethod def setup_run_cfg( - run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, use_cache=True + run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, cache_results=True ) -> BenchRunCfg: run_cfg_out = deepcopy(run_cfg) - run_cfg_out.use_sample_cache = use_cache - run_cfg_out.only_hash_tag = use_cache + run_cfg_out.cache_samples = cache_results + run_cfg_out.only_hash_tag = cache_results run_cfg_out.level = level return run_cfg_out @@ -78,9 +78,9 @@ def run( show: bool = False, save: bool = False, grouped: bool = True, - use_cache: bool = True, + cache_results: bool = True, ) -> List[Bench]: - """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow. If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default use_cache=True so that previous values are reused. + """This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow. If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default cache_results=True so that previous values are reused. Args: min_level (int, optional): The minimum level to start sampling at. Defaults to 2. @@ -93,14 +93,14 @@ def run( show (bool, optional): show the results in the local web browser. Defaults to False. save (bool, optional): save the results to disk in index.html. Defaults to False. grouped (bool, optional): Produce a single html page with all the benchmarks included. Defaults to True. - use_cache (bool, optional): Use the sample cache to reused previous results. Defaults to True. + cache_results (bool, optional): Use the sample cache to reused previous results. Defaults to True. Returns: List[BenchCfg]: A list of bencher instances """ if run_cfg is None: run_cfg = deepcopy(self.run_cfg) - run_cfg = BenchRunner.setup_run_cfg(run_cfg, use_cache=use_cache) + run_cfg = BenchRunner.setup_run_cfg(run_cfg, cache_results=cache_results) if level is not None: min_level = level diff --git a/bencher/bencher.py b/bencher/bencher.py index a6ff20a3..f5d4215e 100644 --- a/bencher/bencher.py +++ b/bencher/bencher.py @@ -327,7 +327,7 @@ def plot_sweep( logging.info("Copy run cfg from bench class") if run_cfg.only_plot: - run_cfg.use_cache = True + run_cfg.cache_results = True self.last_run_cfg = run_cfg @@ -448,7 +448,7 @@ def run_sweep( if run_cfg.clear_cache: c.delete(bench_cfg_hash) logging.info("cleared cache") - elif run_cfg.use_cache: + elif run_cfg.cache_results: logging.info( f"checking for previously calculated results with key: {bench_cfg_hash}" ) @@ -813,7 +813,7 @@ def init_sample_cache(self, run_cfg: BenchRunCfg): cache_name="sample_cache", tag_index=True, size_limit=self.cache_size, - use_cache=run_cfg.use_sample_cache, + cache_results=run_cfg.cache_samples, ) def clear_tag_from_sample_cache(self, tag: str, run_cfg): diff --git a/bencher/example/example_composable_container.py b/bencher/example/example_composable_container.py index dd717e82..d74ad296 100644 --- a/bencher/example/example_composable_container.py +++ b/bencher/example/example_composable_container.py @@ -98,7 +98,7 @@ def example_composable_container_video( if __name__ == "__main__": ex_run_cfg = bch.BenchRunCfg() - ex_run_cfg.use_sample_cache = False + ex_run_cfg.cache_samples = False # ex_run_cfg.level = 2 ex_report = bch.BenchReport() example_composable_container_image(ex_run_cfg, report=ex_report) diff --git a/bencher/example/example_composable_container2.py b/bencher/example/example_composable_container2.py index 769f0505..30dc089e 100644 --- a/bencher/example/example_composable_container2.py +++ b/bencher/example/example_composable_container2.py @@ -144,7 +144,7 @@ def example_composable_container_image( # if __name__ == "__main__": # ex_run_cfg = bch.BenchRunCfg() -# ex_run_cfg.use_sample_cache = False +# ex_run_cfg.cache_samples = False # # ex_run_cfg.level = 2 # ex_report = bch.BenchReport() # example_composable_container_image(ex_run_cfg, report=ex_report) @@ -157,4 +157,4 @@ def example_composable_container_image( # bench_runner.add_run(bench_image) bench_runner.add_run(example_composable_container_image) - bench_runner.run(level=6, show=True, use_cache=False) + bench_runner.run(level=6, show=True, cache_results=False) diff --git a/bencher/example/example_holosweep.py b/bencher/example/example_holosweep.py index 05f80168..240df314 100644 --- a/bencher/example/example_holosweep.py +++ b/bencher/example/example_holosweep.py @@ -95,4 +95,4 @@ def example_holosweep( PlotFunctions().to_gui() bench_run = bch.BenchRunner("bench_runner_test") bench_run.add_run(example_holosweep) - bench_run.run(level=6, show=True, use_cache=False) + bench_run.run(level=6, show=True, cache_results=False) diff --git a/bencher/example/example_image.py b/bencher/example/example_image.py index 7c53ea7e..6103aa76 100644 --- a/bencher/example/example_image.py +++ b/bencher/example/example_image.py @@ -58,7 +58,7 @@ def points_to_polygon_png(self, points: list[float], filename: str): def example_image( run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport() ) -> bch.Bench: - run_cfg.use_cache = False + run_cfg.cache_results = False bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report) bench.result_vars = ["polygon", "area"] @@ -142,7 +142,7 @@ def example_image_vid_sequential( # def example_image_pairs() ex_run_cfg = bch.BenchRunCfg() - ex_run_cfg.use_sample_cache = True + ex_run_cfg.cache_samples = True # ex_run_cfg.debug = True # ex_run_cfg.repeats = 2 ex_run_cfg.level = 4 diff --git a/bencher/example/example_image1.py b/bencher/example/example_image1.py index 4ae3d5ca..46ebe24b 100644 --- a/bencher/example/example_image1.py +++ b/bencher/example/example_image1.py @@ -73,7 +73,7 @@ def example_image_vid_sequential1( if __name__ == "__main__": ex_run_cfg = bch.BenchRunCfg() - ex_run_cfg.use_sample_cache = True + ex_run_cfg.cache_samples = True ex_run_cfg.overwrite_sample_cache = True ex_run_cfg.level = 3 diff --git a/bencher/example/example_sample_cache.py b/bencher/example/example_sample_cache.py index d5e1b0ca..43c3e71a 100644 --- a/bencher/example/example_sample_cache.py +++ b/bencher/example/example_sample_cache.py @@ -2,7 +2,7 @@ class UnreliableClass(bch.ParametrizedSweep): - """This class helps demonstrate benchmarking a function that sometimes crashes during sampling. By using BenchRunCfg.use_sample_cache you can store the results of every call to the benchmark function so data is not lost in the event of a crash. However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data. For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results.""" + """This class helps demonstrate benchmarking a function that sometimes crashes during sampling. By using BenchRunCfg.cache_samples you can store the results of every call to the benchmark function so data is not lost in the event of a crash. However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data. For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results.""" input_val = bch.IntSweep( default=0, @@ -31,7 +31,7 @@ def example_sample_cache( report: bch.BenchReport = bch.BenchReport(), trigger_crash: bool = False, ) -> bch.Bench: - """This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run + """This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run Args: run_cfg (BenchRunCfg): configuration of how to perform the param sweep @@ -50,7 +50,7 @@ def example_sample_cache( title="Example Crashy Function with the sample_cache", input_vars=[UnreliableClass.param.input_val], result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash], - description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""", + description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""", run_cfg=run_cfg, post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed", ) @@ -63,7 +63,7 @@ def example_sample_cache( ex_run_cfg.executor = bch.Executors.SCOOP # this will store the result of of every call to crashy_fn - ex_run_cfg.use_sample_cache = True + ex_run_cfg.cache_samples = True ex_run_cfg.clear_sample_cache = True try: diff --git a/bencher/example/example_sample_cache_context.py b/bencher/example/example_sample_cache_context.py index 473acefb..f68921aa 100644 --- a/bencher/example/example_sample_cache_context.py +++ b/bencher/example/example_sample_cache_context.py @@ -51,7 +51,7 @@ def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_ca def example_cache_context() -> bch.Bench: run_cfg = bch.BenchRunCfg() - run_cfg.use_sample_cache = True + run_cfg.cache_samples = True run_cfg.only_hash_tag = True run_cfg.repeats = 2 run_cfg.parallel = False diff --git a/bencher/example/example_video.py b/bencher/example/example_video.py index 712ec8bb..6574b5c4 100644 --- a/bencher/example/example_video.py +++ b/bencher/example/example_video.py @@ -79,7 +79,7 @@ def example_video( run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport() ) -> bch.Bench: # run_cfg.auto_plot = False - # run_cfg.use_sample_cache = True + # run_cfg.cache_samples = True bench = bch.Bench("example_video", TuringPattern(), run_cfg=run_cfg, report=report) bench.plot_sweep( @@ -111,7 +111,7 @@ def example_video_tap( if __name__ == "__main__": run_cfg_ex = bch.BenchRunCfg() run_cfg_ex.level = 2 - run_cfg_ex.use_sample_cache = True + run_cfg_ex.cache_samples = True run_cfg_ex.only_hash_tag = True # example_video(run_cfg_ex).report.show() diff --git a/bencher/example/experimental/example_hvplot_explorer.py b/bencher/example/experimental/example_hvplot_explorer.py index d4d8097e..95316703 100644 --- a/bencher/example/experimental/example_hvplot_explorer.py +++ b/bencher/example/experimental/example_hvplot_explorer.py @@ -30,7 +30,7 @@ def bench_function(cfg: ExampleBenchCfgIn) -> ExampleBenchCfgOut: post_description="Here you can see the output plot of sin theta between 0 and pi. In the tabs at the top you can also view 3 tabular representations of the data", run_cfg=bch.BenchRunCfg( auto_plot=True, - use_cache=False, + cache_results=False, repeats=2, ), ) diff --git a/bencher/example/shelved/example_float3D_cone.py b/bencher/example/shelved/example_float3D_cone.py index 2955e6cc..54edd07e 100644 --- a/bencher/example/shelved/example_float3D_cone.py +++ b/bencher/example/shelved/example_float3D_cone.py @@ -92,5 +92,5 @@ # if __name__ == "__main__": # ex_run_cfg = bch.BenchRunCfg() -# ex_run_cfg.use_cache = True +# ex_run_cfg.cache_results = True # example_cone(ex_run_cfg).report.show() diff --git a/bencher/job.py b/bencher/job.py index 724c5795..186d9f46 100644 --- a/bencher/job.py +++ b/bencher/job.py @@ -78,11 +78,11 @@ def __init__( cache_name: str = "fcache", tag_index: bool = True, size_limit: int = int(20e9), # 20 GB - use_cache=True, + cache_results=True, ): self.executor_type = executor self.executor = None - if use_cache: + if cache_results: self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit) logging.info(f"cache dir: {self.cache.directory}") else: diff --git a/pyproject.toml b/pyproject.toml index 9b7ee6ab..a48f1e63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "holobench" -version = "1.35.0" +version = "1.36.0" authors = [{ name = "Austin Gregg-Smith", email = "blooop@gmail.com" }] description = "A package for benchmarking the performance of arbitrary functions" diff --git a/test/test_bench_runner.py b/test/test_bench_runner.py index 97e4aa09..95c538c4 100644 --- a/test/test_bench_runner.py +++ b/test/test_bench_runner.py @@ -9,7 +9,7 @@ class TestBenchRunner(unittest.TestCase): # Tests that bch.BenchRunner can be created with default configuration and the import statement in the bch.BenchRunner class is fixed def test_benchrunner_default_configuration_fixed(self): bench_runner = bch.BenchRunner("bench_runner_test") - self.assertEqual(bench_runner.run_cfg.use_sample_cache, True) + self.assertEqual(bench_runner.run_cfg.cache_samples, True) self.assertEqual(bench_runner.run_cfg.only_hash_tag, True) self.assertEqual(bench_runner.run_cfg.level, 2) self.assertEqual(bench_runner.publisher, None) @@ -71,8 +71,8 @@ def run_bench_class(run_cfg: bch.BenchRunCfg, report: bch.BenchReport) -> bch.Be self.assertEqual(results[0].sample_cache.worker_cache_call_count, 0) self.assertEqual(results[0].run_cfg.run_tag, run_tag) - # run with the same tag but set use cache to false, should not hit cache because even tho the tag is the same, use_cache=false - results = bench_runner.run(use_cache=False) + # run with the same tag but set use cache to false, should not hit cache because even tho the tag is the same, cache_results=false + results = bench_runner.run(cache_results=False) self.assertEqual(results[0].sample_cache.worker_wrapper_call_count, 2) self.assertEqual(results[0].sample_cache.worker_fn_call_count, 2) self.assertEqual(results[0].sample_cache.worker_cache_call_count, 0) @@ -144,7 +144,7 @@ def test_benchrunner_repeats(self): # bench_runner.add_run(bench_fn1) # bench_runner.add_run(bench_fn2) # run_cfg = bch.BenchRunCfg() - # run_cfg.use_sample_cache = False + # run_cfg.cache_samples = False # run_cfg.only_hash_tag = False # run_cfg.level = 3 # results = bench_runner.run(run_cfg=run_cfg) diff --git a/test/test_bencher.py b/test/test_bencher.py index f82768c6..e4109dd3 100644 --- a/test/test_bencher.py +++ b/test/test_bencher.py @@ -293,7 +293,7 @@ def test_benching_cache_without_time(self, over_time) -> None: title=title, input_vars=iv, result_vars=rv, - run_cfg=BenchRunCfg(over_time=over_time, use_cache=False, auto_plot=False), + run_cfg=BenchRunCfg(over_time=over_time, cache_results=False, auto_plot=False), ) self.assertEqual( bench2.sample_cache.worker_wrapper_call_count, ExampleBenchCfgIn.param.theta.samples @@ -305,7 +305,7 @@ def test_benching_cache_without_time(self, over_time) -> None: title=title, input_vars=iv, result_vars=rv, - run_cfg=BenchRunCfg(over_time=over_time, use_cache=True, auto_plot=False), + run_cfg=BenchRunCfg(over_time=over_time, cache_results=True, auto_plot=False), ) self.assertEqual( bench2.sample_cache.worker_wrapper_call_count, ExampleBenchCfgIn.param.theta.samples @@ -348,7 +348,7 @@ def test_const_hashing(self, noisy) -> None: const_vars=[ (ExampleBenchCfgIn.param.noisy, noisy), ], - run_cfg=BenchRunCfg(use_cache=True, auto_plot=False), + run_cfg=BenchRunCfg(cache_results=True, auto_plot=False), ) # the result should be cached so the call count should be the same as before self.assertEqual( diff --git a/test/test_sample_cache.py b/test/test_sample_cache.py index ab4cb73b..c4f99e38 100644 --- a/test/test_sample_cache.py +++ b/test/test_sample_cache.py @@ -13,7 +13,7 @@ def call_bencher(self, bencher, run_cfg): UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash, ], - description="""This example shows how to use the use_sample_cache option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashing during the run""", + description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashing during the run""", run_cfg=run_cfg, post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed", tag="testing_tag3", @@ -29,7 +29,7 @@ def sample_cache(self): run_cfg.repeats = 1 run_cfg.executor = bch.Executors.SERIAL # THE ASSERTS WILL ONLY WORK IF RUN SERIALLY!!! - run_cfg.use_sample_cache = True # this will store the result of every call + run_cfg.cache_samples = True # this will store the result of every call run_cfg.only_hash_tag = True run_cfg.auto_plot = False