From 1df4011f9783d3c40a7a1d5b42aafe2d5f24e9d3 Mon Sep 17 00:00:00 2001 From: Xinyuan Huang Date: Fri, 12 Jun 2020 18:04:05 -0700 Subject: [PATCH 1/5] Update version to 0.7.0-rc2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 81e56e16..13706a34 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.0-rc1 \ No newline at end of file +0.7.0-rc2 From 93e77f79704dfe6351d783ab8c4330b8a8f1c839 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Tue, 16 Jun 2020 03:16:54 +0200 Subject: [PATCH 2/5] [compliance_checker][minigo] fixing a typo in the target accuracy check according to https://github.com/mlperf/training_policies/blob/16302f66575f2afbaa096eba5bdc26b62918fd3f/training_rules.adoc#3-benchmarks target is 0.5 not 0.55 --- mlperf_logging/compliance_checker/0.7.0/minigo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlperf_logging/compliance_checker/0.7.0/minigo.yaml b/mlperf_logging/compliance_checker/0.7.0/minigo.yaml index a97cd200..3c8748bd 100644 --- a/mlperf_logging/compliance_checker/0.7.0/minigo.yaml +++ b/mlperf_logging/compliance_checker/0.7.0/minigo.yaml @@ -27,7 +27,7 @@ REQ: AT_LEAST_ONE CHECK: - "'epoch_num' in v['metadata']" - ATLEAST_ONE_CHECK: "v['value'] >= 0.55 and v['value'] < 1.0" + ATLEAST_ONE_CHECK: "v['value'] >= 0.5 and v['value'] < 1.0" - KEY: NAME: block_start From 1235de8125f5ba3f170d4d393931e8411bf230d1 Mon Sep 17 00:00:00 2001 From: Victor Bittorf Date: Tue, 16 Jun 2020 18:56:00 -0700 Subject: [PATCH 3/5] Adjusting package verifier to support convergence dropping. (#45) * Adjusting package verifier to support convergence dropping. * Fixing error that prevents multiple frameworks from printing. * Fixing typo. --- .../compliance_checker/mlp_compliance.py | 13 ++-- .../package_checker/package_checker.py | 14 +++-- .../result_summarizer/result_summarizer.py | 60 +++++++++++++++---- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/mlperf_logging/compliance_checker/mlp_compliance.py b/mlperf_logging/compliance_checker/mlp_compliance.py index 2f5d46c4..b57fa4a4 100644 --- a/mlperf_logging/compliance_checker/mlp_compliance.py +++ b/mlperf_logging/compliance_checker/mlp_compliance.py @@ -76,12 +76,13 @@ def overwrite_messages(self, keys): def log_messages(self): message_separator = '\n' + '-' * 30 + '\n' - - print(message_separator.join([ - *self.warnings.values(), - *self.overwritable.values(), - *self.not_overwritable - ])) + message = message_separator.join([ + *self.warnings.values(), + *self.overwritable.values(), + *self.not_overwritable + ]) + if message: + print(message) def has_messages(self): return self.not_overwritable or self.overwritable diff --git a/mlperf_logging/package_checker/package_checker.py b/mlperf_logging/package_checker/package_checker.py index 3d319594..344bf79e 100644 --- a/mlperf_logging/package_checker/package_checker.py +++ b/mlperf_logging/package_checker/package_checker.py @@ -53,8 +53,7 @@ def check_training_result_files(folder, ruleset, quiet, werror): ruleset: The ruleset such as 0.6.0 or 0.7.0. """ - errors_found = 0 - + too_many_errors = False result_folder = os.path.join(folder, 'results') for system_folder in _get_sub_folders(result_folder): for benchmark_folder in _get_sub_folders(system_folder): @@ -88,6 +87,7 @@ def check_training_result_files(folder, ruleset, quiet, werror): _EXPECTED_RESULT_FILE_COUNTS[benchmark], len(result_files))) + errors_found = 0 result_files.sort() for result_file in result_files: result_basename = os.path.basename(result_file) @@ -107,11 +107,15 @@ def check_training_result_files(folder, ruleset, quiet, werror): valid, _, _, _ = mlp_compliance.main(result_file, config_file, checker) if not valid: errors_found += 1 - + if errors_found == 1: + print('WARNING: One file does not comply.') + print('WARNING: Allowing this failure under olympic scoring rules.') + if errors_found > 1: + too_many_errors = True _print_divider_bar() - if errors_found > 0: - raise Exception('Found errors in logging, see log above for details.') + if too_many_errors: + raise Exception('Found too many errors in logging, see log above for details.') def check_training_package(folder, ruleset, quiet, werror): diff --git a/mlperf_logging/result_summarizer/result_summarizer.py b/mlperf_logging/result_summarizer/result_summarizer.py index c133efaa..9221f8a4 100644 --- a/mlperf_logging/result_summarizer/result_summarizer.py +++ b/mlperf_logging/result_summarizer/result_summarizer.py @@ -12,6 +12,8 @@ import re import sys +from ..compliance_checker import mlp_compliance + _ALLOWED_BENCHMARKS_V06 = [ 'resnet', 'ssd', @@ -106,17 +108,27 @@ def _code_url(system_desc, ruleset): def _row_key(system_desc): - system_name = system_desc['system_name'] + system_name = '{}-{}'.format(system_desc['system_name'], system_desc['framework']) if system_name == 'tpu-v3': chips = int(system_desc['accelerators_per_node']) * 2 return 'tpu-v3-{:04d}'.format(chips) return system_name -def _read_mlperf_score(result_file): +def _read_mlperf_score(result_file, ruleset): with open(result_file, 'r') as f: result = f.read() + config_file = '{ruleset}/common.yaml'.format(ruleset=ruleset) + checker = mlp_compliance.make_checker( + ruleset=ruleset, + quiet=True, + werror=False) + valid, _, _, _ = mlp_compliance.main(result_file, config_file, checker) + + if not valid: + return None + run_start = re.search(_RUN_START_REGEX, result) if run_start is None: raise Exception('Failed to match run_start!.') @@ -129,11 +141,25 @@ def _read_mlperf_score(result_file): return minutes -def _compute_olympic_average(scores): - copied_scores = copy.deepcopy(scores) - copied_scores.sort() - copied_scores = copied_scores[1:-1] - return sum(copied_scores) / len(copied_scores) +def _compute_olympic_average(scores, dropped_scores): + """There are two possible cases we might handle: + If dropped_scores == 0, then we compute a normal olympiic score. + If dropped_scores > 0, then the maximum was already dropped + (and does not appear in scores) because it did not converge. + """ + sum_of_scores = sum(scores) + count = len(scores) + + # Subtract off the min + sum_of_scores -= min(scores) + count -= 1 + + # Subtract off the max, only if the max was not already dropped + if dropped_scores == 0: + sum_of_scores -= max(scores) + count -= 1 + + return sum_of_scores * 1.0 / count def _is_organization_folder(folder): @@ -216,11 +242,23 @@ def summarize_results(folder, ruleset): pattern = '{folder}/result_*.txt'.format(folder=benchmark_folder) result_files = glob.glob(pattern, recursive=True) scores = [] + dropped_scores = 0 for result_file in result_files: - score = _read_mlperf_score(result_file) - scores.append(score) - - benchmark_scores[benchmark] = _compute_olympic_average(scores) + score = _read_mlperf_score(result_file, ruleset) + if score is None: + dropped_scores += 1 + else: + scores.append(score) + if dropped_scores > 1: + print('CRITICAL ERROR: Too many non-converging runs for {} {}/{}'. + format(desc['submitter'], system, benchmark)) + print('** CRITICAL ERROR ** Results in the table for {} {}/{} are NOT correct'. + format(desc['submitter'], system, benchmark)) + if dropped_scores == 1: + print('NOTICE: Dropping non-converged run for {} {}/{} using olympic scoring.' + .format(desc['submitter'], system, benchmark)) + + benchmark_scores[benchmark] = _compute_olympic_average(scores, dropped_scores) # Construct scores portion of the row. if ruleset == '0.6.0': From 37fdf7e2c0566f31298416f22e31d3f32b56c479 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 18 Jun 2020 19:59:16 +0200 Subject: [PATCH 4/5] [compliance_checker][minig] remove broken checks (#34) --- .../compliance_checker/0.7.0_warn/minigo.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/mlperf_logging/compliance_checker/0.7.0_warn/minigo.yaml b/mlperf_logging/compliance_checker/0.7.0_warn/minigo.yaml index f92314a3..ad90be1f 100644 --- a/mlperf_logging/compliance_checker/0.7.0_warn/minigo.yaml +++ b/mlperf_logging/compliance_checker/0.7.0_warn/minigo.yaml @@ -2,12 +2,6 @@ - KEY: NAME: save_model REQ: AT_LEAST_ONE - CHECK: - - "s['in_epoch']" - - "v['value']['iteration'] == s['last_epoch']" - - "not s['model_saved']" - POST: " s['model_saved'] = True ; s['save_model_ts'].append(ll.timestamp) " - - KEY: NAME: global_batch_size @@ -24,11 +18,6 @@ REQ: EXACTLY_ONE CHECK: " len(v['value']) > 0" -- KEY: - NAME: opt_base_learning_rate - REQ: EXACTLY_ONE - CHECK: " len(v['value']) > 0 " - - KEY: NAME: opt_weight_decay REQ: EXACTLY_ONE @@ -94,10 +83,3 @@ REQ: EXACTLY_ONE CHECK: " v['value'] > 0 " -# block_start/stop keys are not required in minigo, so re-define without REQ -- KEY: - NAME: block_start - -- KEY: - NAME: block_stop - From 82cd16f58f348aaa0067364ade8be7c6c36f8c6e Mon Sep 17 00:00:00 2001 From: "David M. Chen" <51184357+davidmochen@users.noreply.github.com> Date: Thu, 18 Jun 2020 10:59:40 -0700 Subject: [PATCH 5/5] Split 0.7.0 yaml files into open and closed variants (#46) * Split 0.7.0 yaml into open and closed variants When division = open or closed, the open or closed yaml variant is enqueued. The closed yaml files are branched from the previous yaml files. The open yaml files have the hyperparameter checks removed and only the accuracy checks retained. Tested: with some submission packages * Rename .yaml to open_.yaml For better naming consistency with the closed_.yaml files. --- .../0.7.0/{bert.yaml => closed_bert.yaml} | 0 .../compliance_checker/0.7.0/closed_common.yaml | 6 ++++++ .../0.7.0/{dlrm.yaml => closed_dlrm.yaml} | 0 .../0.7.0/{gnmt.yaml => closed_gnmt.yaml} | 0 .../0.7.0/{maskrcnn.yaml => closed_maskrcnn.yaml} | 0 .../0.7.0/{minigo.yaml => closed_minigo.yaml} | 0 .../0.7.0/{resnet.yaml => closed_resnet.yaml} | 2 +- .../{resnet_lars.yaml => closed_resnet_lars.yaml} | 0 .../{resnet_sgd.yaml => closed_resnet_sgd.yaml} | 0 .../0.7.0/{ssd.yaml => closed_ssd.yaml} | 0 .../{transformer.yaml => closed_transformer.yaml} | 0 mlperf_logging/compliance_checker/0.7.0/common.yaml | 2 +- .../compliance_checker/0.7.0/open_bert.yaml | 7 +++++++ .../compliance_checker/0.7.0/open_common.yaml | 6 ++++++ .../compliance_checker/0.7.0/open_dlrm.yaml | 7 +++++++ .../compliance_checker/0.7.0/open_gnmt.yaml | 8 ++++++++ .../compliance_checker/0.7.0/open_maskrcnn.yaml | 12 ++++++++++++ .../compliance_checker/0.7.0/open_minigo.yaml | 9 +++++++++ .../compliance_checker/0.7.0/open_resnet.yaml | 7 +++++++ .../compliance_checker/0.7.0/open_ssd.yaml | 7 +++++++ .../compliance_checker/0.7.0/open_transformer.yaml | 7 +++++++ 21 files changed, 78 insertions(+), 2 deletions(-) rename mlperf_logging/compliance_checker/0.7.0/{bert.yaml => closed_bert.yaml} (100%) create mode 100644 mlperf_logging/compliance_checker/0.7.0/closed_common.yaml rename mlperf_logging/compliance_checker/0.7.0/{dlrm.yaml => closed_dlrm.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{gnmt.yaml => closed_gnmt.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{maskrcnn.yaml => closed_maskrcnn.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{minigo.yaml => closed_minigo.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{resnet.yaml => closed_resnet.yaml} (81%) rename mlperf_logging/compliance_checker/0.7.0/{resnet_lars.yaml => closed_resnet_lars.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{resnet_sgd.yaml => closed_resnet_sgd.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{ssd.yaml => closed_ssd.yaml} (100%) rename mlperf_logging/compliance_checker/0.7.0/{transformer.yaml => closed_transformer.yaml} (100%) create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_bert.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_common.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_dlrm.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_gnmt.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_maskrcnn.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_minigo.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_resnet.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_ssd.yaml create mode 100644 mlperf_logging/compliance_checker/0.7.0/open_transformer.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/bert.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_bert.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/bert.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_bert.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/closed_common.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_common.yaml new file mode 100644 index 00000000..fb8917ba --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/closed_common.yaml @@ -0,0 +1,6 @@ + +- KEY: + NAME: submission_benchmark + REQ: EXACTLY_ONE + CHECK: " v['value'] in ['resnet', 'ssd', 'maskrcnn', 'transformer', 'gnmt', 'minigo', 'dlrm', 'bert'] " + POST: " enqueue_config('0.7.0/closed_{}.yaml'.format(v['value'])) " diff --git a/mlperf_logging/compliance_checker/0.7.0/dlrm.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_dlrm.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/dlrm.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_dlrm.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/gnmt.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_gnmt.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/gnmt.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_gnmt.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/maskrcnn.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_maskrcnn.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/maskrcnn.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_maskrcnn.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/minigo.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_minigo.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/minigo.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_minigo.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/resnet.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_resnet.yaml similarity index 81% rename from mlperf_logging/compliance_checker/0.7.0/resnet.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_resnet.yaml index 20d52532..2135d219 100644 --- a/mlperf_logging/compliance_checker/0.7.0/resnet.yaml +++ b/mlperf_logging/compliance_checker/0.7.0/closed_resnet.yaml @@ -8,7 +8,7 @@ NAME: opt_name REQ: EXACTLY_ONE CHECK: " v['value'] in ['sgd', 'lars'] " - POST: " enqueue_config('0.7.0/resnet_{}.yaml'.format(v['value'])) " + POST: " enqueue_config('0.7.0/closed_resnet_{}.yaml'.format(v['value'])) " - KEY: NAME: eval_accuracy diff --git a/mlperf_logging/compliance_checker/0.7.0/resnet_lars.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_resnet_lars.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/resnet_lars.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_resnet_lars.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/resnet_sgd.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_resnet_sgd.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/resnet_sgd.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_resnet_sgd.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/ssd.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_ssd.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/ssd.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_ssd.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/transformer.yaml b/mlperf_logging/compliance_checker/0.7.0/closed_transformer.yaml similarity index 100% rename from mlperf_logging/compliance_checker/0.7.0/transformer.yaml rename to mlperf_logging/compliance_checker/0.7.0/closed_transformer.yaml diff --git a/mlperf_logging/compliance_checker/0.7.0/common.yaml b/mlperf_logging/compliance_checker/0.7.0/common.yaml index cc4d689d..57ce0239 100644 --- a/mlperf_logging/compliance_checker/0.7.0/common.yaml +++ b/mlperf_logging/compliance_checker/0.7.0/common.yaml @@ -41,6 +41,7 @@ NAME: submission_division REQ: EXACTLY_ONE CHECK: " v['value'] in ['closed', 'open'] " + POST: " enqueue_config('0.7.0/{}_common.yaml'.format(v['value'])) " - KEY: NAME: submission_status @@ -51,7 +52,6 @@ NAME: submission_benchmark REQ: EXACTLY_ONE CHECK: " v['value'] in ['resnet', 'ssd', 'maskrcnn', 'transformer', 'gnmt', 'minigo', 'dlrm', 'bert'] " - POST: " enqueue_config('0.7.0/{}.yaml'.format(v['value'])) " # at least one record should be found, but any found records must pass the test - KEY: diff --git a/mlperf_logging/compliance_checker/0.7.0/open_bert.yaml b/mlperf_logging/compliance_checker/0.7.0/open_bert.yaml new file mode 100644 index 00000000..5c5dbcae --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_bert.yaml @@ -0,0 +1,7 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.712 and v['value'] < 1.0" diff --git a/mlperf_logging/compliance_checker/0.7.0/open_common.yaml b/mlperf_logging/compliance_checker/0.7.0/open_common.yaml new file mode 100644 index 00000000..efc9b289 --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_common.yaml @@ -0,0 +1,6 @@ + +- KEY: + NAME: submission_benchmark + REQ: EXACTLY_ONE + CHECK: " v['value'] in ['resnet', 'ssd', 'maskrcnn', 'transformer', 'gnmt', 'minigo', 'dlrm', 'bert'] " + POST: " enqueue_config('0.7.0/open_{}.yaml'.format(v['value'])) " diff --git a/mlperf_logging/compliance_checker/0.7.0/open_dlrm.yaml b/mlperf_logging/compliance_checker/0.7.0/open_dlrm.yaml new file mode 100644 index 00000000..a7757431 --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_dlrm.yaml @@ -0,0 +1,7 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.8025 and v['value'] < 1.0" diff --git a/mlperf_logging/compliance_checker/0.7.0/open_gnmt.yaml b/mlperf_logging/compliance_checker/0.7.0/open_gnmt.yaml new file mode 100644 index 00000000..6d12a7b7 --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_gnmt.yaml @@ -0,0 +1,8 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.240 and v['value'] < 1" + diff --git a/mlperf_logging/compliance_checker/0.7.0/open_maskrcnn.yaml b/mlperf_logging/compliance_checker/0.7.0/open_maskrcnn.yaml new file mode 100644 index 00000000..169dc912 --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_maskrcnn.yaml @@ -0,0 +1,12 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'BBOX' in v['value']" + - "'SEGM' in v['value']" + - "'epoch_num' in v['metadata']" + POST: " s['accuracy_printed'] = True " + ATLEAST_ONE_CHECK: "v['value']['BBOX'] >= 0.377 and v['value']['BBOX'] < 1.0 and v['value']['SEGM'] >= 0.339 and v['value']['SEGM'] < 1.0" + + diff --git a/mlperf_logging/compliance_checker/0.7.0/open_minigo.yaml b/mlperf_logging/compliance_checker/0.7.0/open_minigo.yaml new file mode 100644 index 00000000..c376647a --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_minigo.yaml @@ -0,0 +1,9 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.5 and v['value'] < 1.0" + + diff --git a/mlperf_logging/compliance_checker/0.7.0/open_resnet.yaml b/mlperf_logging/compliance_checker/0.7.0/open_resnet.yaml new file mode 100644 index 00000000..3636beaf --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_resnet.yaml @@ -0,0 +1,7 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.7590 and v['value'] < 1.0" diff --git a/mlperf_logging/compliance_checker/0.7.0/open_ssd.yaml b/mlperf_logging/compliance_checker/0.7.0/open_ssd.yaml new file mode 100644 index 00000000..e7bc6fe2 --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_ssd.yaml @@ -0,0 +1,7 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.230 and v['value'] < 1.0" diff --git a/mlperf_logging/compliance_checker/0.7.0/open_transformer.yaml b/mlperf_logging/compliance_checker/0.7.0/open_transformer.yaml new file mode 100644 index 00000000..889634d7 --- /dev/null +++ b/mlperf_logging/compliance_checker/0.7.0/open_transformer.yaml @@ -0,0 +1,7 @@ + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'epoch_num' in v['metadata']" + ATLEAST_ONE_CHECK: "v['value'] >= 0.2500 and v['value'] < 1"