diff --git a/.github/workflows/test-mlperf-inference-gptj.yml b/.github/workflows/test-mlperf-inference-gptj.yml index 755d74adac..040d00f9b0 100644 --- a/.github/workflows/test-mlperf-inference-gptj.yml +++ b/.github/workflows/test-mlperf-inference-gptj.yml @@ -4,8 +4,8 @@ name: MLPerf inference GPT-J on: - pull_request: - branches: [ "main1", "dev1" ] + push: + branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-gptj.yml' - '**' @@ -13,8 +13,7 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: [ self-hosted, linux, x64 ] strategy: fail-fast: false matrix: @@ -23,15 +22,12 @@ jobs: precision: [ "bfloat16" ] steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install cmind + source gh_action/bin/deactivate || python3 -m venv gh_action + source gh_action/bin/activate + export CM_REPOS=$HOME/GH_CM cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference GPTJ run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="cTuning" --model=gptj --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet + cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean diff --git a/automation/script/module.py b/automation/script/module.py index 46181a087c..e40cf8b353 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1,6 +1,6 @@ # # CM "script" automation helps users to encode their MLOps, DevOps and other knowledge -# as portable and reusable automation recipes with simple tags, native scripts +# as portable and reusable automation recipes with simple tags, native scripts # and a unified CLI, Python API and JSON/YAML meta descriptions. # # This is a stable prototype of the CM script automation being developed by Grigori Fursin and Arjun Suresh @@ -18,7 +18,7 @@ class CAutomation(Automation): """ - CM "script" automation actions + CM "script" automation actions (making native scripts more portable, deterministic, reusable and reproducible) """ @@ -59,13 +59,13 @@ def __init__(self, cmind, automation_file): 'CM_GIT_*', 'CM_RENEW_CACHE_ENTRY'] - self.input_flags_converted_to_tmp_env = ['path'] + self.input_flags_converted_to_tmp_env = ['path'] - self.input_flags_converted_to_env = ['input', - 'output', - 'name', - 'extra_cache_tags', - 'skip_compile', + self.input_flags_converted_to_env = ['input', + 'output', + 'name', + 'extra_cache_tags', + 'skip_compile', 'skip_run', 'accept_license', 'skip_system_deps', @@ -81,7 +81,7 @@ def run(self, i): Run CM script Args: - (CM input dict): + (CM input dict): (out) (str): if 'con', output to console @@ -146,7 +146,7 @@ def run(self, i): (fake_deps) (bool): if True, will fake run the dependent scripts (run_state) (dict): Internal run state - (debug_script_tags) (str): if !='', run cmd/bash before executing a native command + (debug_script_tags) (str): if !='', run cmd/bash before executing a native command inside a script specified by these tags (debug_script) (bool): if True, debug current script (set debug_script_tags to the tags of a current script) @@ -168,11 +168,11 @@ def run(self, i): (pause) (bool): if True, pause at the end of the main script (Press Enter to continue) - (repro) (bool): if True, dump cm-run-script-input.json, cm-run_script_output.json, + (repro) (bool): if True, dump cm-run-script-input.json, cm-run_script_output.json, cm-run-script-state.json, cm-run-script-info.json to improve the reproducibility of results - (repro_prefix) (str): if !='', use it to record above files {repro-prefix)-input.json ... + (repro_prefix) (str): if !='', use it to record above files {repro-prefix)-input.json ... (repro_dir) (str): if !='', use this directory to dump info (default = 'cm-repro') (dump_version_info) (bool): dump info about resolved versions of tools in dependencies @@ -183,7 +183,7 @@ def run(self, i): (script_call_prefix) (str): how to call script in logs and READMEs (cm run script) - (skip_sys_utils) (bool): if True, set env['CM_SKIP_SYS_UTILS']='yes' + (skip_sys_utils) (bool): if True, set env['CM_SKIP_SYS_UTILS']='yes' to skip CM sys installation (skip_sudo) (bool): if True, set env['CM_TMP_SKIP_SUDO']='yes' to let scripts deal with that @@ -231,7 +231,7 @@ def _run(self, i): if repro_prefix == '': repro_prefix = 'cm-run-script' repro_dir = i.get('repro_dir', '') - if repro_dir == '': + if repro_dir == '': repro_dir = os.path.join(os.getcwd(), 'cm-repro') if not os.path.isdir(repro_dir): os.makedirs(repro_dir) @@ -240,7 +240,7 @@ def _run(self, i): if repro_prefix!='': dump_repro_start(repro_prefix, i) - + recursion = i.get('recursion', False) # If first script run, check if can write to current directory @@ -257,7 +257,7 @@ def _run(self, i): if len(script_input)>0: utils.merge_dicts({'dict1':i, 'dict2':script_input}) - + recursion_int = int(i.get('recursion_int',0))+1 @@ -353,7 +353,7 @@ def _run(self, i): if debug_uid!='': r = _update_env(env, 'CM_TMP_DEBUG_UID', debug_uid) if r['return']>0: return r - + fake_deps = i.get('fake_deps', False) if fake_deps: env['CM_TMP_FAKE_DEPS']='yes' @@ -377,16 +377,16 @@ def _run(self, i): if not silent: silent = True if str(i.get('s', '')).lower() in ['true', 'yes', 'on'] else False - + if silent: if 'verbose' in i: del(i['verbose']) if 'v' in i: del(i['v']) env['CM_TMP_SILENT']='yes' run_state['tmp_silent']=True - + if 'verbose' in i: verbose=i['verbose'] elif 'v' in i: verbose=i['v'] - + if verbose: env['CM_VERBOSE']='yes' run_state['tmp_verbose']=True @@ -475,7 +475,7 @@ def _run(self, i): env['CM_' + key.upper()] = value r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r ############################################################################################################ @@ -508,7 +508,7 @@ def _run(self, i): r = self.search(ii) if r['return']>0: return r - # Search function will return + # Search function will return list_of_found_scripts = r['list'] @@ -520,7 +520,7 @@ def _run(self, i): # # Print what was searched! # cm_script_info = 'CM script' # -# x = 'with' +# x = 'with' # if parsed_script_alias !='' : # cm_script_info += ' '+x+' alias "{}"'.format(parsed_script_alias) # x = 'and' @@ -579,9 +579,9 @@ def _run(self, i): if len(list_of_found_scripts) == 0: return {'return':16, 'error':'no scripts were found with above tags and variations\n'+r.get('warning', '')} - # Sometimes there is an ambiguity when someone adds a script + # Sometimes there is an ambiguity when someone adds a script # while duplicating a UID. In such case, we will return >1 script - # and will start searching in the cache ... + # and will start searching in the cache ... # We are detecing such cases here: if len(list_of_found_scripts)>1 and script_tags_string=='' and parsed_script_alias!='' and '?' not in parsed_script_alias and '*' not in parsed_script_alias: x='Ambiguity in the following scripts have the same UID - please change that in _cm.json or _cm.yaml:\n' @@ -590,7 +590,7 @@ def _run(self, i): return {'return':1, 'error':x} - # STEP 100 Output: list_of_found_scripts based on tags (with variations) and/or parsed_artifact + # STEP 100 Output: list_of_found_scripts based on tags (with variations) and/or parsed_artifact # script_tags [] - contains tags without variations (starting from _ such as _cuda) # variation_tags [] - contains only variations tags (without _) # string_tags_string [str] (joined script_tags) @@ -625,7 +625,7 @@ def _run(self, i): - # STEP 300: If more than one CM script found (example: "get compiler"), + # STEP 300: If more than one CM script found (example: "get compiler"), # first, check if selection was already remembered! # second, check in cache to prune scripts @@ -806,7 +806,7 @@ def _run(self, i): # STEP 500 output: script_artifact - unique selected script artifact - # (cache_list) pruned for the unique script if cache is used + # (cache_list) pruned for the unique script if cache is used # meta - script meta # path - script path # found_script_tags [] - all tags of the found script @@ -825,7 +825,7 @@ def _run(self, i): # HERE WE HAVE ORIGINAL ENV - # STEP 600: Continue updating env + # STEP 600: Continue updating env # Add default env from meta to new env if not empty # (env NO OVERWRITE) script_artifact_default_env = meta.get('default_env',{}) @@ -871,7 +871,7 @@ def _run(self, i): # STEP 800: Process variations and update env (overwrite from env and update form default_env) - # VARIATIONS HAS THE PRIORITY OVER + # VARIATIONS HAS THE PRIORITY OVER # MULTIPLE VARIATIONS (THAT CAN BE TURNED ON AT THE SAME TIME) SHOULD NOT HAVE CONFLICTING ENV # VARIATIONS OVERWRITE current ENV but not input keys (they become const) @@ -896,7 +896,7 @@ def _run(self, i): # USE CASE: # HERE we may have versions in script input and env['CM_VERSION_*'] - # STEP 900: Get version, min, max, usable from env (priority if passed from another script to force version), + # STEP 900: Get version, min, max, usable from env (priority if passed from another script to force version), # then script input, then script meta # VERSIONS SHOULD NOT BE USED INSIDE VARIATIONS (in meta)! @@ -930,11 +930,11 @@ def _run(self, i): key = version_index[1] note = version_index[2] - if version_value !='': + if version_value !='': env[key] = version_value notes.append(note.format(version_value)) -# elif key in env: +# elif key in env: # # If version_X is "", remove related key from ENV ... # del(env[key]) @@ -958,16 +958,16 @@ def _run(self, i): if adr: self._merge_dicts_with_tags(add_deps_recursive, adr) #Processing them again using updated deps for add_deps_recursive - r = update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive) + r = update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive, env) + - # STEP 1100: Update deps from input r = update_deps_from_input(deps, post_deps, prehook_deps, posthook_deps, i) if r['return']>0: return r r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r if str(env.get('CM_RUN_STATE_DOCKER', False)).lower() in ['true', '1', 'yes']: if state.get('docker'): @@ -1001,8 +1001,7 @@ def _run(self, i): if env.get('CM_NAME','')!='': extra_cache_tags.append('name-'+env['CM_NAME'].strip().lower()) - - + ############################################################################################################ # Check if need to clean output files clean_output_files = meta.get('clean_output_files', []) @@ -1106,7 +1105,7 @@ def _run(self, i): if len(deps)>0: logging.debug(recursion_spaces + ' - Checking dynamic dependencies on other CM scripts:') - r = self._call_run_deps(deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, True, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1114,13 +1113,13 @@ def _run(self, i): logging.debug(recursion_spaces + ' - Processing env after dependencies ...') r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r # Check chain of prehook dependencies on other CM scripts. (No execution of customize.py for cached scripts) logging.debug(recursion_spaces + ' - Checking prehook dependencies on other CM scripts:') - r = self._call_run_deps(prehook_deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(prehook_deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1170,7 +1169,7 @@ def _run(self, i): clean_env_keys_post_deps = meta.get('clean_env_keys_post_deps',[]) - r = self._call_run_deps(posthook_deps, self.local_env_keys, clean_env_keys_post_deps, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(posthook_deps, self.local_env_keys, clean_env_keys_post_deps, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1178,7 +1177,7 @@ def _run(self, i): logging.debug(recursion_spaces + ' - Checking post dependencies on other CM scripts:') # Check chain of post dependencies on other CM scripts - r = self._call_run_deps(post_deps, self.local_env_keys, clean_env_keys_post_deps, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(post_deps, self.local_env_keys, clean_env_keys_post_deps, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1191,12 +1190,12 @@ def _run(self, i): # Add more tags to cached tags # based on meta information of the found script x = 'script-artifact-' + meta['uid'] - if x not in cached_tags: + if x not in cached_tags: cached_tags.append(x) # Add all tags from the original CM script for x in meta.get('tags', []): - if x not in cached_tags: + if x not in cached_tags: cached_tags.append(x) @@ -1208,7 +1207,7 @@ def _run(self, i): # Finalize tmp tags tmp_tags += [ t for t in cached_tags if not t.startswith("-") ] - # Check if some variations are missing + # Check if some variations are missing # though it should not happen! for t in variation_tags: if t.startswith("-"): @@ -1221,7 +1220,7 @@ def _run(self, i): logging.debug(recursion_spaces+' - Creating new "cache" script artifact in the CM local repository ...') logging.debug(recursion_spaces+' - Tags: {}'.format(','.join(tmp_tags))) - if version != '': + if version != '': cached_meta['version'] = version ii = {'action':'update', @@ -1271,8 +1270,8 @@ def _run(self, i): env['CM_RENEW_CACHE_ENTRY']='yes' # Prepare files to be cleaned - clean_files = [self.tmp_file_run_state, - self.tmp_file_run_env, + clean_files = [self.tmp_file_run_state, + self.tmp_file_run_env, self.tmp_file_ver, self.tmp_file_env + bat_ext, self.tmp_file_env_all + bat_ext, @@ -1282,11 +1281,11 @@ def _run(self, i): if not found_cached and len(meta.get('clean_files', [])) >0: clean_files = meta['clean_files'] + clean_files - ################################ + ################################ if not found_cached: if len(warnings)>0: logging.warn('=================================================') - logging.warn('WARNINGS:') + logging.warn('WARNINGS:') for w in warnings: logging.warn(' '+w) logging.warn('=================================================') @@ -1315,7 +1314,7 @@ def _run(self, i): if ry['return']>0: return ry if ry['comparison'] > 0: - if version_max_usable!='': + if version_max_usable!='': version = version_max_usable else: version = version_max @@ -1337,7 +1336,7 @@ def _run(self, i): r = _update_env(env, 'CM_TMP_CURRENT_SCRIPT_PATH', path) if r['return']>0: return r - + # Run chain of docker dependencies if current run cmd is from inside a docker container docker_deps = [] if i.get('docker_run_deps'): @@ -1351,7 +1350,7 @@ def _run(self, i): logging.debug(recursion_spaces + ' - Checking docker run dependencies on other CM scripts:') - r = self._call_run_deps(docker_deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(docker_deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, False, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1359,13 +1358,13 @@ def _run(self, i): logging.debug(recursion_spaces + ' - Processing env after docker run dependencies ...') r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r # Check chain of dependencies on other CM scripts if len(deps)>0: logging.debug(recursion_spaces + ' - Checking dependencies on other CM scripts:') - r = self._call_run_deps(deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, False, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1373,7 +1372,7 @@ def _run(self, i): logging.debug(recursion_spaces + ' - Processing env after dependencies ...') r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r # Clean some output files @@ -1523,7 +1522,7 @@ def _run(self, i): if len(r.get('add_extra_cache_tags',[]))>0: for t in r['add_extra_cache_tags']: if t not in cached_tags: - cached_tags.append(t) + cached_tags.append(t) if print_env: @@ -1535,7 +1534,7 @@ def _run(self, i): if len(prehook_deps)>0: logging.debug(recursion_spaces + ' - Checking prehook dependencies on other CM scripts:') - r = self._call_run_deps(prehook_deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, + r = self._call_run_deps(prehook_deps, self.local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r @@ -1563,7 +1562,7 @@ def _run(self, i): if len(r.get('add_extra_cache_tags',[]))>0 and cache: for t in r['add_extra_cache_tags']: if t not in cached_tags: - cached_tags.append(t) + cached_tags.append(t) # Check chain of post dependencies on other CM scripts clean_env_keys_post_deps = meta.get('clean_env_keys_post_deps',[]) @@ -1582,7 +1581,7 @@ def _run(self, i): if v!='': for t in v.split(','): x = 'deps-' + prefix + t - if x not in cached_tags: + if x not in cached_tags: cached_tags.append(x) @@ -1618,7 +1617,7 @@ def _run(self, i): # Restore original env/state and merge env/state - # This is needed since we want to keep original env/state outside this script + # This is needed since we want to keep original env/state outside this script # If we delete env and create a new dict, the original one outside this script will be detached # That's why we just clean all keys in original env/state (used oustide) # And then copy saved_env (with new_env merged) and saved_state (with new_state merged) @@ -1642,7 +1641,7 @@ def _run(self, i): # Check if need to remove tag if remove_tmp_tag: # Save state, env and deps for reuse - r = utils.save_json(file_name = os.path.join(cached_path, self.file_with_cached_state), + r = utils.save_json(file_name = os.path.join(cached_path, self.file_with_cached_state), meta={'new_state':new_state, 'new_env':new_env, 'deps':deps, 'version': version}) if r['return']>0: return r @@ -1723,7 +1722,7 @@ def _run(self, i): tmp_run_file = 'tmp-run{}'.format(x[3]) if os.path.isfile(tmp_run_file): xtmp_run_file = 'Change and run "{}". '.format(tmp_run_file) - + env_script.append('echo {}Running debug shell. {}Type exit to quit ...{}\n'.format(x[2], xtmp_run_file, x[2])) env_script.append('echo{}\n'.format(x[1])) env_script.append('\n') @@ -1816,14 +1815,14 @@ def _run(self, i): return r rr = {'return':0, 'env':env, 'new_env':new_env, 'state':state, 'new_state':new_state, 'deps': run_state['deps']} - + # Print output as json to console if i.get('json', False) or i.get('j', False): import json logging.info(json.dumps(rr, indent=2)) - + # Check if save json to file if repro_prefix !='': @@ -1910,7 +1909,7 @@ def _dump_version_info_for_script(self, output_dir = os.getcwd(), quiet = False, pass for f in ['cm-run-script-versions.json', 'version_info.json']: if not quiet and not silent: - logging.info('Dumping versions to {}'.format(f)) + logging.info('Dumping versions to {}'.format(f)) r = utils.save_json(f, self.run_state.get('version_info', [])) if r['return']>0: return r @@ -2069,7 +2068,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env if x not in warnings: warnings.append(x) #Processing them again using updated deps for add_deps_recursive - r = update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive) + r = update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive, env) if r['return']>0: return r if len(required_disk_space)>0: @@ -2222,7 +2221,7 @@ def version(self, i): (out) (str): if 'con', output to console - Returns: + Returns: (CM return dict): * return (int): return code == 0 if no error and >0 if error @@ -2258,9 +2257,9 @@ def search(self, i): if 'parsed_artifact' in i: del(i['parsed_artifact']) # Force substitute tags i['tags']=artifact.replace(' ',',') - + ############################################################################################################ - # Process tags to find script(s) and separate variations + # Process tags to find script(s) and separate variations # (not needed to find scripts) tags_string = i.get('tags','').strip() @@ -2432,7 +2431,7 @@ def native_run(self, i): Add CM script Args: - (CM input dict): + (CM input dict): env (dict): environment command (str): string @@ -2453,7 +2452,7 @@ def native_run(self, i): # Create temporary script name script_name = i.get('script_name','') - if script_name=='': + if script_name=='': script_name='tmp-native-run.' if os.name == 'nt': @@ -2487,7 +2486,7 @@ def native_run(self, i): script.append('') - # Add CMD + # Add CMD script.append(cmd) # Record script @@ -2505,7 +2504,7 @@ def add(self, i): Add CM script Args: - (CM input dict): + (CM input dict): (out) (str): if 'con', output to console @@ -2866,7 +2865,7 @@ def _process_variation_tags_in_groups(script, variation_tags, groups, excluded_v ############################################################################## def _call_run_deps(script, deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state, - add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags='', + add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags='', verbose=False, show_time=False, extra_recursion_spaces=' ', run_state={'deps':[], 'fake_deps':[], 'parent': None}): if len(deps) == 0: return {'return': 0} @@ -2881,15 +2880,15 @@ def _call_run_deps(script, deps, local_env_keys, local_env_keys_from_meta, env, local_env_keys += local_env_keys_from_meta r = script._run_deps(deps, local_env_keys, env, state, const, const_state, add_deps_recursive, recursion_spaces, - remembered_selections, variation_tags_string, found_cached, debug_script_tags, + remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, extra_recursion_spaces, run_state) if r['return']>0: return r return {'return': 0} ############################################################################## - def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, add_deps_recursive, recursion_spaces, - remembered_selections, variation_tags_string='', from_cache=False, debug_script_tags='', + def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, add_deps_recursive, recursion_spaces, + remembered_selections, variation_tags_string='', from_cache=False, debug_script_tags='', verbose=False, show_time=False, extra_recursion_spaces=' ', run_state={'deps':[], 'fake_deps':[], 'parent': None}): """ Runs all the enabled dependencies and pass them env minus local env @@ -2906,25 +2905,8 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a if not d.get('tags'): continue - if d.get('skip_if_fake_run', False) and env.get('CM_TMP_FAKE_RUN','')=='yes': + if is_dep_tobe_skipped(d, env): continue - - if "enable_if_env" in d: - if not enable_or_skip_script(d["enable_if_env"], env): - continue - - if "enable_if_any_env" in d: - if not any_enable_or_skip_script(d["enable_if_any_env"], env): - continue - - if "skip_if_env" in d: - if enable_or_skip_script(d["skip_if_env"], env): - continue - - if "skip_if_any_env" in d: - x = any_enable_or_skip_script(d["skip_if_any_env"], env) - if x: - continue if from_cache and not d.get("dynamic", None): continue @@ -3108,7 +3090,7 @@ def _get_readme(self, cmd_parts, run_state): pip install cmind -U ``` -Check [this readme](https://github.com/mlcommons/ck/blob/master/docs/installation.md) +Check [this readme](https://github.com/mlcommons/ck/blob/master/docs/installation.md) with more details about installing CM and dependencies across different platforms (Ubuntu, MacOS, Windows, RHEL, ...). @@ -3424,7 +3406,7 @@ def find_file_in_paths(self, i): duplicate = True break except Exception as e: - # This function fails on Windows sometimes + # This function fails on Windows sometimes # because some files can't be accessed pass @@ -3534,7 +3516,7 @@ def detect_version_using_script(self, i): Detect version using script Args: - (CM input dict): + (CM input dict): (recursion_spaces) (str): add space to print @@ -3584,8 +3566,8 @@ def detect_version_using_script(self, i): run_script_input['recursion_spaces'] = recursion_spaces - if rx['return'] == 0: - # Version was detected + if rx['return'] == 0: + # Version was detected detected_version = rx.get('version','') if detected_version != '': @@ -3607,7 +3589,7 @@ def find_artifact(self, i): Find some artifact (file) by name Args: - (CM input dict): + (CM input dict): file_name (str): filename to find @@ -3618,7 +3600,7 @@ def find_artifact(self, i): (env_path) (str): env key to pass path to the script to detect version (run_script_input) (dict): use this input to run script to detect version - (default_path_env_key) (str): check in default paths from global env + (default_path_env_key) (str): check in default paths from global env (PATH, PYTHONPATH, LD_LIBRARY_PATH ...) (recursion_spaces) (str): add space to print @@ -3634,7 +3616,7 @@ def find_artifact(self, i): found_path (list): found path to an artifact full_path (str): full path to a found artifact - default_path_list (list): list of default paths + default_path_list (list): list of default paths """ @@ -3660,7 +3642,7 @@ def find_artifact(self, i): hook = i.get('hook', None) - # Check if forced to search in a specific path or multiple paths + # Check if forced to search in a specific path or multiple paths # separated by OS var separator (usually : or ;) path = env.get('CM_TMP_PATH','') @@ -3712,7 +3694,7 @@ def find_artifact(self, i): # Prepare paths to search r = self.find_file_in_paths({'paths': path_list, - 'file_name': file_name, + 'file_name': file_name, 'select': True, 'select_default': select_default, 'detect_version': i.get('detect_version', False), @@ -3753,7 +3735,7 @@ def find_artifact(self, i): if env_path_key != '': env[env_path_key] = file_path - return {'return':0, 'found_path':found_path, + return {'return':0, 'found_path':found_path, 'found_file_path':file_path, 'found_file_name':os.path.basename(file_path), 'default_path_list': default_path_list} @@ -3863,7 +3845,7 @@ def parse_version(self, i): Parse version (used in post processing functions) Args: - (CM input dict): + (CM input dict): (file_name) (str): filename to get version from (tmp-ver.out by default) match_text (str): RE match text string @@ -3893,11 +3875,11 @@ def parse_version(self, i): debug = i.get('debug', False) r = utils.load_txt(file_name = file_name, - check_if_exists = True, + check_if_exists = True, split = True, match_text = match_text, fail_if_no_match = 'version was not detected') - if r['return']>0: + if r['return']>0: if r.get('string','')!='': r['error'] += ' ({})'.format(r['string']) return r @@ -3927,7 +3909,8 @@ def update_deps(self, i): deps = i['deps'] add_deps = i['update_deps'] - update_deps(deps, add_deps, False) + env = i.get('env', {}) + update_deps(deps, add_deps, False, env) return {'return':0} @@ -3948,7 +3931,7 @@ def doc(self, i): Document CM script. Args: - (CM input dict): + (CM input dict): (out) (str): if 'con', output to console @@ -3991,7 +3974,7 @@ def gui(self, i): if artifact != '': if ' ' in artifact: tags = artifact.replace(' ',',') - + if tags=='': tags = i.get('tags','') @@ -4070,7 +4053,7 @@ def _available_variations(self, i): return error with available variations Args: - (CM input dict): + (CM input dict): meta (dict): meta of the script @@ -4137,7 +4120,7 @@ def find_cached_script(i): """ import copy - + recursion_spaces = i['recursion_spaces'] script_tags = i['script_tags'] cached_tags = [] @@ -4173,7 +4156,7 @@ def find_cached_script(i): if len(found_script_tags)>0: for x in found_script_tags: - if x not in cached_tags: + if x not in cached_tags: cached_tags.append(x) explicit_cached_tags=copy.deepcopy(cached_tags) @@ -4182,7 +4165,7 @@ def find_cached_script(i): explicit_variation_tags_string = '' for t in explicit_variation_tags: - if explicit_variation_tags_string != '': + if explicit_variation_tags_string != '': explicit_variation_tags_string += ',' if t.startswith("-"): x = "-_" + t[1:] @@ -4190,16 +4173,16 @@ def find_cached_script(i): x = '_' + t explicit_variation_tags_string += x - if x not in explicit_cached_tags: + if x not in explicit_cached_tags: explicit_cached_tags.append(x) logging.debug(recursion_spaces+' - Prepared explicit variations: {}'.format(explicit_variation_tags_string)) - + if len(variation_tags)>0: variation_tags_string = '' for t in variation_tags: - if variation_tags_string != '': + if variation_tags_string != '': variation_tags_string += ',' if t.startswith("-"): x = "-_" + t[1:] @@ -4207,21 +4190,21 @@ def find_cached_script(i): x = '_' + t variation_tags_string += x - if x not in cached_tags: + if x not in cached_tags: cached_tags.append(x) logging.debug(recursion_spaces+' - Prepared variations: {}'.format(variation_tags_string)) # Add version if version !='': - if 'version-'+version not in cached_tags: + if 'version-'+version not in cached_tags: cached_tags.append('version-'+version) explicit_cached_tags.append('version-'+version) # Add extra cache tags (such as "virtual" for python) if len(extra_cache_tags)>0: for t in extra_cache_tags: - if t not in cached_tags: + if t not in cached_tags: cached_tags.append(t) explicit_cached_tags.append(t) @@ -4235,14 +4218,14 @@ def find_cached_script(i): if v!='': for t in v.split(','): x = 'deps-' + prefix + t - if x not in cached_tags: + if x not in cached_tags: cached_tags.append(x) explicit_cached_tags.append(x) # Check if already cached if not new_cache_entry: search_tags = '-tmp' - if len(cached_tags) >0 : + if len(cached_tags) >0 : search_tags += ',' + ','.join(explicit_cached_tags) logging.debug(recursion_spaces+' - Searching for cached script outputs with the following tags: {}'.format(search_tags)) @@ -4355,7 +4338,7 @@ def any_enable_or_skip_script(meta, env): found = True elif value in meta_key: found = True - + # If found any match from the list (OR), return if found: return True @@ -4364,18 +4347,19 @@ def any_enable_or_skip_script(meta, env): ############################################################################################################ def _update_env(env, key=None, value=None): + if key == None or value == None: return {'return': 1, 'error': 'None value not expected in key and value arguments in _update_env.'} if not isinstance(key, str): return {'return': 1, 'error': 'String value expected inside key argument.'} - + env[key] = value r = update_env_with_values(env) - if r['return']>0: return r + if r['return']>0: return r return {'return': 0} - + ############################################################################################################ def update_env_with_values(env, fail_on_not_found=False, extra_env={}): """ @@ -4525,7 +4509,7 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): # Script name was added by user - we need to check that it really exists (on Linux or Windows) check_if_run_script_exists = True if script_name == '': - # Here is the default script name - if it doesn't exist, we skip it. + # Here is the default script name - if it doesn't exist, we skip it. # However, if it's explicitly specified, we check it and report # if it's missing ... script_name = 'run' @@ -4650,28 +4634,28 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): if script_repo_git and script_repo_alias!='': repo_to_report = 'https://github.com/'+script_repo_alias.replace('@','/')+'/issues' - + if repo_to_report == '': repo_to_report = 'https://github.com/mlcommons/cm4mlops/issues' note = ''' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Note that it is often a portability issue of a third-party tool or a native script +Note that it is often a portability issue of a third-party tool or a native script wrapped and unified by this CM script (automation recipe). Please re-run this script with --repro flag and report this issue with the original command line, cm-repro directory and full log here: {} -The CM concept is to collaboratively fix such issues inside portable CM scripts -to make existing tools and native scripts more portable, interoperable +The CM concept is to collaboratively fix such issues inside portable CM scripts +to make existing tools and native scripts more portable, interoperable and deterministic. Thank you'''.format(repo_to_report) rr = {'return':2, 'error':'Portable CM script failed (name = {}, return code = {})\n\n{}'.format(meta['alias'], rc, note)} if repro_prefix != '': dump_repro(repro_prefix, rr, run_state) - + return rr # Load updated state if exists @@ -4694,12 +4678,12 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): updated_env = r['dict'] utils.merge_dicts({'dict1':env, 'dict2':updated_env, 'append_lists':True, 'append_unique':True}) - + if postprocess != '' and customize_code is not None and postprocess in dir(customize_code): if not run_state.get('tmp_silent', False): logging.info(recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__)) - + if len(posthook_deps)>0 and (postprocess == "postprocess"): r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, ' ', run_state) @@ -4754,7 +4738,7 @@ def run_postprocess(customize_code, customize_common_input, recursion_spaces, en ii['meta'] = meta if run_script_input != None: - ii['run_script_input'] = run_script_input + ii['run_script_input'] = run_script_input r = customize_code.postprocess(ii) return r @@ -4833,7 +4817,7 @@ def convert_env_to_script(env, os_info, start_script = []): env_value = xx if type(env_value)==list else xx[0] - # Process special env + # Process special env key = k if k.startswith('+'): @@ -4841,7 +4825,7 @@ def convert_env_to_script(env, os_info, start_script = []): key=k[1:] first = key[0] env_separator = os_info['env_separator'] - # If key starts with a symbol use it as the list separator (+ CFLAG will use ' ' the + # If key starts with a symbol use it as the list separator (+ CFLAG will use ' ' the # list separator while +;TEMP will use ';' as the separator) if not first.isalnum(): env_separator = first @@ -4892,37 +4876,88 @@ def clean_tmp_files(clean_files, recursion_spaces): return {'return':0} +############################################################################## + +def update_dynamic_env_values(mydict, env): + """ + Internal: update the dynamic value in given dict identified by <<<...>>> with the value from env dictionary if set + """ + import re + # Regular expression to match <<>> + pattern = re.compile(r'<<<(.*?)>>>') + + def replace_variables(value): + # Function to replace the <<>> with corresponding value from env + if isinstance(value, str): # Only process if the value is a string + matches = pattern.findall(value) + for match in matches: + if match in env: # Replace only if the variable is in env + value = value.replace(f'<<<{match}>>>', str(env[match])) + return value + + # Recursively update the dictionary + for key, val in mydict.items(): + if isinstance(val, dict): + # If the value is a dictionary, recurse into it + update_dynamic_env_values(val, env) + else: + # Replace variables in the current value + mydict[key] = replace_variables(val) + + return + + ############################################################################## def update_dep_info(dep, new_info): """ Internal: add additional info to a dependency """ for info in new_info: + if info == "tags": tags = dep.get('tags', '') tags_list = tags.split(",") new_tags_list = new_info["tags"].split(",") - combined_tags = tags_list + list(set(new_tags_list) - set(tags_list)) + filtered_new_tags_list = [ i for i in new_tags_list if "<<<" not in i] + combined_tags = tags_list + list(set(filtered_new_tags_list) - set(tags_list)) dep['tags'] = ",".join(combined_tags) + + elif "enable_if_" in info or "skip_if_" in info: #These are meant for adding ad/adr and not for the dependency + continue + elif type(new_info[info]) == dict: + if not dep.get(info, {}): + dep[info] = new_info[info] + elif type(dep[info]) == dict: + utils.merge_dicts({'dict1':dep[info], 'dict2':new_info[info], 'append_lists':True, 'append_unique':True}) + #else: Throw error? + elif type(new_info[info]) == list: + if not dep.get(info, []): + dep[info] = new_info[info] + elif type(dep[info]) == list: + dep[info] += new_info[info] + #else: Throw error? else: dep[info] = new_info[info] ############################################################################## -def update_deps(deps, add_deps, fail_error=False): +def update_deps(deps, add_deps, fail_error=False, env={}): """ Internal: add deps tags, version etc. by name """ - #deps_info_to_add = [ "version", "version_min", "version_max", "version_max_usable", "path" ] + #deps_info_to_add = [ "version", "version_min", "version_max", "version_max_usable", "path", "tags", .... ] new_deps_info = {} - for new_deps_name in add_deps: + for new_dep_name in add_deps: + if is_dep_tobe_skipped(add_deps[new_dep_name], env): + continue dep_found = False for dep in deps: names = dep.get('names',[]) - if new_deps_name in names: - update_dep_info(dep, add_deps[new_deps_name]) + if new_dep_name in names: + update_dynamic_env_values(add_deps[new_dep_name], env) + update_dep_info(dep, add_deps[new_dep_name]) dep_found = True if not dep_found and fail_error: - return {'return':1, 'error':new_deps_name + ' is not one of the dependency'} + return {'return':1, 'error':new_dep_name + ' is not one of the dependency'} return {'return':0} @@ -4961,6 +4996,32 @@ def append_deps(deps, new_deps): return {'return':0} +############################################################################## +def is_dep_tobe_skipped(d, env): + """ + Internal: check if this dependency is to be skipped + """ + if d.get('skip_if_fake_run', False) and env.get('CM_TMP_FAKE_RUN','')=='yes': + return True + + if "enable_if_env" in d: + if not enable_or_skip_script(d["enable_if_env"], env): + return True + + if "enable_if_any_env" in d: + if not any_enable_or_skip_script(d["enable_if_any_env"], env): + return True + + if "skip_if_env" in d: + if enable_or_skip_script(d["skip_if_env"], env): + return True + + if "skip_if_any_env" in d: + if any_enable_or_skip_script(d["skip_if_any_env"], env): + return True + + return False + ############################################################################## def update_deps_from_input(deps, post_deps, prehook_deps, posthook_deps, i): """ @@ -4978,17 +5039,19 @@ def update_deps_from_input(deps, post_deps, prehook_deps, posthook_deps, i): else: utils.merge_dicts({'dict1':add_deps_recursive_info_from_input, 'dict2':i.get('add_deps_recursive', {}), 'append_lists':True, 'append_unique':True}) + env = i.get('env', {}) + if add_deps_info_from_input: - r1 = update_deps(deps, add_deps_info_from_input, True) - r2 = update_deps(post_deps, add_deps_info_from_input, True) - r3 = update_deps(prehook_deps, add_deps_info_from_input, True) - r4 = update_deps(posthook_deps, add_deps_info_from_input, True) + r1 = update_deps(deps, add_deps_info_from_input, True, env) + r2 = update_deps(post_deps, add_deps_info_from_input, True, env) + r3 = update_deps(prehook_deps, add_deps_info_from_input, True, env) + r4 = update_deps(posthook_deps, add_deps_info_from_input, True, env) if r1['return']>0 and r2['return']>0 and r3['return']>0 and r4['return']>0: return r1 if add_deps_recursive_info_from_input: - update_deps(deps, add_deps_recursive_info_from_input) - update_deps(post_deps, add_deps_recursive_info_from_input) - update_deps(prehook_deps, add_deps_recursive_info_from_input) - update_deps(posthook_deps, add_deps_recursive_info_from_input) + update_deps(deps, add_deps_recursive_info_from_input, False, env) + update_deps(post_deps, add_deps_recursive_info_from_input, False, env) + update_deps(prehook_deps, add_deps_recursive_info_from_input, False, env) + update_deps(posthook_deps, add_deps_recursive_info_from_input, False, env) return {'return':0} @@ -5039,10 +5102,10 @@ def update_state_from_meta(meta, env, state, deps, post_deps, prehook_deps, post else: utils.merge_dicts({'dict1':add_deps_info, 'dict2':meta.get('add_deps', {}), 'append_lists':True, 'append_unique':True}) if add_deps_info: - r1 = update_deps(deps, add_deps_info, True) - r2 = update_deps(post_deps, add_deps_info, True) - r3 = update_deps(prehook_deps, add_deps_info, True) - r4 = update_deps(posthook_deps, add_deps_info, True) + r1 = update_deps(deps, add_deps_info, True, env) + r2 = update_deps(post_deps, add_deps_info, True, env) + r3 = update_deps(prehook_deps, add_deps_info, True, env) + r4 = update_deps(posthook_deps, add_deps_info, True, env) if r1['return']>0 and r2['return']>0 and r3['return'] > 0 and r4['return'] > 0: return r1 input_mapping = meta.get('input_mapping', {}) @@ -5066,7 +5129,7 @@ def update_state_from_meta(meta, env, state, deps, post_deps, prehook_deps, post # utils.merge_dicts({'dict1':docker_input_mapping, 'dict2':new_docker_input_mapping, 'append_lists':True, 'append_unique':True}) utils.merge_dicts({'dict1':docker_settings, 'dict2':new_docker_settings, 'append_lists':True, 'append_unique':True}) if docker_settings.get('deps', []): - update_deps(docker_settings['deps'], add_deps_info, False) + update_deps(docker_settings['deps'], add_deps_info, False, env) state['docker'] = docker_settings new_env_keys_from_meta = meta.get('new_env_keys', []) @@ -5080,15 +5143,15 @@ def update_state_from_meta(meta, env, state, deps, post_deps, prehook_deps, post return {'return':0} ############################################################################## -def update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive_info): +def update_adr_from_meta(deps, post_deps, prehook_deps, posthook_deps, add_deps_recursive_info, env={}): """ Internal: update add_deps_recursive from meta """ if add_deps_recursive_info: - update_deps(deps, add_deps_recursive_info) - update_deps(post_deps, add_deps_recursive_info) - update_deps(prehook_deps, add_deps_recursive_info) - update_deps(posthook_deps, add_deps_recursive_info) + update_deps(deps, add_deps_recursive_info, False, env) + update_deps(post_deps, add_deps_recursive_info, False, env) + update_deps(prehook_deps, add_deps_recursive_info, False, env) + update_deps(posthook_deps, add_deps_recursive_info, False, env) return {'return':0} @@ -5312,7 +5375,7 @@ def dump_repro_start(repro_prefix, ii): info['run_uid'] = r['uid'] r = utils.get_current_date_time({}) - if r['return']==0: + if r['return']==0: info['run_iso_datetime'] = r['iso_datetime'] with open(repro_prefix+'-info.json', 'w', encoding='utf-8') as f: diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index efbc922652..b193ff5dfe 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1524,7 +1524,7 @@ def dockerfile(i): if i.get('print_deps'): cm_input = {'action': 'run', 'automation': 'script', - 'tags': f'{tag_string}', + 'tags': f"""{i.get('tags')}""", 'print_deps': True, 'quiet': True, 'silent': True, @@ -1557,7 +1557,7 @@ def dockerfile(i): 'fake_run_option': fake_run_option, 'comments': comments, 'run_cmd': f'{run_cmd} --quiet', - 'script_tags': f'{tag_string}', + 'script_tags': f"""{i.get('tags')}""", 'copy_files': docker_copy_files, 'quiet': True, 'env': env, @@ -2037,11 +2037,11 @@ def docker(i): 'image_repo': image_repo, 'interactive': interactive, 'mounts': mounts, - 'image_name': 'cm-script-'+script_alias, + 'image_name': i.get('docker_image_name', ''), # 'image_tag': script_alias, 'image_tag_extra': image_tag_extra, 'detached': detached, - 'script_tags': f'{tag_string}', + 'script_tags': f"""{i.get('tags')}""", 'run_cmd': final_run_cmd, 'v': i.get('v', False), 'quiet': True, diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 29e7183064..5ecb69b5d5 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -533,6 +533,7 @@ deps: - tags: get,dataset,coco2014,_validation names: - coco2014-preprocessed + - coco2014-dataset enable_if_env: CM_MODEL: - stable-diffusion-xl @@ -590,6 +591,9 @@ deps: - tags: generate,user-conf,mlperf,inference names: - user-conf-generator + skip_if_env: + CM_RUN_STATE_DOCKER: + - 'yes' # Install MLPerf loadgen - tags: get,loadgen @@ -1232,7 +1236,7 @@ variations: float16: group: precision add_deps_recursive: - ml-model: + ml-model-float16: tags: _fp16 env: @@ -1243,9 +1247,9 @@ variations: bfloat16: group: precision add_deps_recursive: - ml-model: + ml-model-bfloat16: tags: - _fp16 + _fp32 env: CM_MLPERF_QUANTIZATION: off CM_MLPERF_MODEL_PRECISION: bfloat16 diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 9a8f9ced26..9683f7f625 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -42,7 +42,6 @@ input_mapping: mlperf_conf: CM_MLPERF_CONF mode: CM_MLPERF_LOADGEN_MODE output_dir: CM_MLPERF_OUTPUT_DIR - performance_sample_count: CM_MLPERF_PERFORMANCE_SAMPLE_COUNT scenario: CM_MLPERF_LOADGEN_SCENARIO user_conf: CM_MLPERF_USER_CONF devices: CM_MLPERF_NVIDIA_HARNESS_DEVICES @@ -477,6 +476,11 @@ variations: names: - scipy version: 1.10.1 + - tags: get,generic-python-lib,_package.numpy + names: + - numpy + version_max: 1.22.99 + version_max_usable: "1.22" sdxl,v4.1: deps: diff --git a/script/app-mlperf-inference-nvidia/customize.py b/script/app-mlperf-inference-nvidia/customize.py index f775789aaf..b176cb60ff 100644 --- a/script/app-mlperf-inference-nvidia/customize.py +++ b/script/app-mlperf-inference-nvidia/customize.py @@ -361,7 +361,7 @@ def preprocess(i): if input_format: run_config += f" --input_format={input_format}" - performance_sample_count = env.get('CM_MLPERF_PERFORMANCE_SAMPLE_COUNT') + performance_sample_count = env.get('CM_MLPERF_LOADGEN_PERFORMANCE_SAMPLE_COUNT') if performance_sample_count: run_config += f" --performance_sample_count={performance_sample_count}" diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 15a46a7a98..e55c1cdf08 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -1565,15 +1565,21 @@ docker: - tags: get,mlperf,inference,results,dir,local names: - get-mlperf-inference-results-dir + skip_if_env: + OUTPUT_BASE_DIR: [ on ] - tags: get,mlperf,inference,submission,dir,local names: - get-mlperf-inference-submission-dir + skip_if_env: + CM_MLPERF_INFERENCE_SUBMISSION_DIR: [ on ] + pre_run_cmds: #- cm pull repo && cm run script --tags=get,git,repo,_repo.https://github.com/GATEOverflow/inference_results_v4.0.git --update - cm pull repo mounts: - "${{ CM_DATASET_IMAGENET_PATH }}:${{ CM_DATASET_IMAGENET_PATH }}" - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}" + - "${{ OUTPUT_BASE_DIR }}:${{ OUTPUT_BASE_DIR }}" - "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}" - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" - "${{ CM_CRITEO_PREPROCESSED_PATH }}:${{ CM_CRITEO_PREPROCESSED_PATH }}" diff --git a/script/build-docker-image/_cm.yaml b/script/build-docker-image/_cm.yaml index 9c958874dc..a9dc8cb677 100644 --- a/script/build-docker-image/_cm.yaml +++ b/script/build-docker-image/_cm.yaml @@ -42,5 +42,4 @@ prehook_deps: - enable_if_env: CM_BUILD_DOCKERFILE: - 'yes' - - '1' tags: build,dockerfile diff --git a/script/build-docker-image/customize.py b/script/build-docker-image/customize.py index 90684e036d..e66eddbd8b 100644 --- a/script/build-docker-image/customize.py +++ b/script/build-docker-image/customize.py @@ -15,6 +15,7 @@ def preprocess(i): else: build_dockerfile = True env['CM_BUILD_DOCKERFILE'] = "yes" + env['CM_DOCKERFILE_BUILD_FROM_IMAGE_SCRIPT'] = "yes" CM_DOCKER_BUILD_ARGS = env.get('+ CM_DOCKER_BUILD_ARGS', []) @@ -34,55 +35,54 @@ def preprocess(i): # else: # env['CM_BUILD_DOCKERFILE'] = "no" # - if "CM_DOCKER_IMAGE_REPO" not in env: + if env.get("CM_DOCKER_IMAGE_REPO", "") == '': env['CM_DOCKER_IMAGE_REPO'] = "local" docker_image_name = env.get('CM_DOCKER_IMAGE_NAME', '') if docker_image_name == '': - docker_image_name = env.get('CM_DOCKER_RUN_SCRIPT_TAGS','').replace(',', '-').replace('_','') - if docker_image_name == '': - docker_image_name = 'cm' - - env['CM_DOCKER_IMAGE_NAME'] = docker_image_name + docker_image_name = "cm-script-" +env.get('CM_DOCKER_RUN_SCRIPT_TAGS','').replace(',', '-').replace('_','-') + env['CM_DOCKER_IMAGE_NAME'] = docker_image_name if env.get("CM_DOCKER_IMAGE_TAG", "") == '': env['CM_DOCKER_IMAGE_TAG'] = "latest" - if env.get("CM_DOCKER_CACHE", "yes") in ["no", "False", False]: + if str(env.get("CM_DOCKER_CACHE", "yes")).lower() in ["no", "false", "0"]: env["CM_DOCKER_CACHE_ARG"] = " --no-cache" CMD = '' image_name = get_image_name(env) - if not build_dockerfile: - # Write .dockerignore - with open('.dockerignore', 'w') as f: - f.write('.git\n') + if build_dockerfile: + dockerfile_path = "\${CM_DOCKERFILE_WITH_PATH}" - # Prepare CMD to build image - XCMD = [ - 'docker build ' + env.get('CM_DOCKER_CACHE_ARG',''), - ' ' + build_args, - ' -f "' + dockerfile_path + '"', - ' -t "' + image_name, - ' .' - ] + # Write .dockerignore + with open('.dockerignore', 'w') as f: + f.write('.git\n') - with open(dockerfile_path + '.build.sh', 'w') as f: - f.write(' \\\n'.join(XCMD) + '\n') + # Prepare CMD to build image + XCMD = [ + 'docker build ' + env.get('CM_DOCKER_CACHE_ARG',''), + ' ' + build_args, + ' -f "' + dockerfile_path + '"', + ' -t "' + image_name, + ' .' + ] - with open(dockerfile_path + '.build.bat', 'w') as f: - f.write(' ^\n'.join(XCMD) + '\n') + with open(dockerfile_path + '.build.sh', 'w') as f: + f.write(' \\\n'.join(XCMD) + '\n') - CMD = ''.join(XCMD) + with open(dockerfile_path + '.build.bat', 'w') as f: + f.write(' ^\n'.join(XCMD) + '\n') - print ('================================================') - print ('CM generated the following Docker build command:') - print ('') - print (CMD) + CMD = ''.join(XCMD) - print ('') + print ('================================================') + print ('CM generated the following Docker build command:') + print ('') + print (CMD) + + print ('') env['CM_DOCKER_BUILD_CMD'] = CMD diff --git a/script/build-dockerfile/_cm.yaml b/script/build-dockerfile/_cm.yaml index 74a6a528a2..5436c6ddd3 100644 --- a/script/build-dockerfile/_cm.yaml +++ b/script/build-dockerfile/_cm.yaml @@ -56,7 +56,6 @@ post_deps: - enable_if_env: CM_BUILD_DOCKER_IMAGE: - 'yes' - - '1' names: - build-docker-image tags: build,docker,image diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 5b2209a37a..b5bae64fc3 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -21,7 +21,7 @@ def preprocess(i): input_args = [] copy_files = [] - if 'CM_DOCKER_RUN_SCRIPT_TAGS' in env: + if env.get('CM_DOCKER_RUN_SCRIPT_TAGS', '') != '': script_tags=env['CM_DOCKER_RUN_SCRIPT_TAGS'] found_scripts = cm.access({'action': 'search', 'automation': 'script', 'tags': script_tags}) scripts_list = found_scripts['list'] @@ -62,7 +62,7 @@ def preprocess(i): else: cm_mlops_repo_branch_string = "" - if 'CM_DOCKERFILE_WITH_PATH' not in env: + if env.get('CM_DOCKERFILE_WITH_PATH', '') == '': env['CM_DOCKERFILE_WITH_PATH'] = os.path.join(os.getcwd(), "Dockerfile") dockerfile_with_path = env['CM_DOCKERFILE_WITH_PATH'] @@ -180,9 +180,10 @@ def preprocess(i): f.write(EOL+'# Install python packages' + EOL) python = get_value(env, config, 'PYTHON', 'CM_DOCKERFILE_PYTHON') - f.write('RUN {} -m venv cm-venv'.format(python) + " " + EOL) - f.write('RUN . cm-venv/bin/activate' + EOL) - f.write('RUN {} -m pip install --user '.format(python) + " ".join(get_value(env, config, 'python-packages')) + ' ' + pip_extra_flags + ' ' + EOL) + f.write('RUN {} -m venv /home/cmuser/venv/cm'.format(python) + " " + EOL) + f.write('ENV PATH="/home/cmuser/venv/cm/bin:$PATH"' + EOL) + #f.write('RUN . /opt/venv/cm/bin/activate' + EOL) + f.write('RUN {} -m pip install '.format(python) + " ".join(get_value(env, config, 'python-packages')) + ' ' + pip_extra_flags + ' ' + EOL) f.write(EOL+'# Download CM repo for scripts' + EOL) @@ -260,6 +261,8 @@ def preprocess(i): s = r['string'] f.write(s + EOL) + print(f"""Dockerfile written at {dockerfile_with_path}""") + f.close() #f = open(env['CM_DOCKERFILE_WITH_PATH'], "r") diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/_cm.yaml b/script/clean-nvidia-mlperf-inference-scratch-space/_cm.yaml new file mode 100644 index 0000000000..8d74e1e727 --- /dev/null +++ b/script/clean-nvidia-mlperf-inference-scratch-space/_cm.yaml @@ -0,0 +1,43 @@ +alias: clean-nvidia-mlperf-inference-scratch-space +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: false +tags: +- clean +- nvidia +- scratch +- space +- mlperf +- inference +uid: bb41f6e3608e4e8a +deps: + # Get Nvidia scratch space where data and models get downloaded + - tags: get,mlperf,inference,nvidia,scratch,space + names: + - nvidia-scratch-space + +variations: + sdxl: + group: model + env: + CM_MODEL: sdxl + downloaded-data: + group: artifact + env: + CM_CLEAN_ARTIFACT_NAME: downloaded_data + preprocessed-data: + group: artifact + env: + CM_CLEAN_ARTIFACT_NAME: preprocessed_data + downloaded-model: + group: artifact + env: + CM_CLEAN_ARTIFACT_NAME: downloaded_model + v4.1: + group: version + env: + CM_NVIDIA_MLPERF_INFERENCE_CODE_VERSION: v4.1 + v4.0: + group: version + env: + CM_NVIDIA_MLPERF_INFERENCE_CODE_VERSION: v4.0 diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/customize.py b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py new file mode 100644 index 0000000000..d12f9b3e1d --- /dev/null +++ b/script/clean-nvidia-mlperf-inference-scratch-space/customize.py @@ -0,0 +1,22 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + return {'return':0} diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/run.bat b/script/clean-nvidia-mlperf-inference-scratch-space/run.bat new file mode 100644 index 0000000000..648302ca71 --- /dev/null +++ b/script/clean-nvidia-mlperf-inference-scratch-space/run.bat @@ -0,0 +1 @@ +rem native script diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/run.sh b/script/clean-nvidia-mlperf-inference-scratch-space/run.sh new file mode 100644 index 0000000000..4c23c380ea --- /dev/null +++ b/script/clean-nvidia-mlperf-inference-scratch-space/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency + +echo "Running: " +echo "${CM_RUN_CMD}" +echo "" + +if [[ ${CM_FAKE_RUN} != "yes" ]]; then + eval "${CM_RUN_CMD}" + test $? -eq 0 || exit 1 +fi diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index cdcf1e7f52..b1b77ed4ba 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -37,16 +37,14 @@ def generate_submission(i): inp=i['input'] if env.get('CM_MLPERF_INFERENCE_RESULTS_DIR_', '') == '': - env['CM_MLPERF_INFERENCE_RESULTS_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_DIR'], f"{env['CM_MLPERF_RUN_STYLE']}_results") + results_dir = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_DIR'], f"{env['CM_MLPERF_RUN_STYLE']}_results") else: - env['CM_MLPERF_INFERENCE_RESULTS_DIR'] = env['CM_MLPERF_INFERENCE_RESULTS_DIR_'] + results_dir = env['CM_MLPERF_INFERENCE_RESULTS_DIR_'] mlperf_path = env['CM_MLPERF_INFERENCE_SOURCE'] submission_checker_dir = os.path.join(mlperf_path, "tools", "submission") sys.path.append(submission_checker_dir) - results_dir = env['CM_MLPERF_INFERENCE_RESULTS_DIR'] - if env.get('CM_MLPERF_INFERENCE_SUBMISSION_DIR', '') == '': from pathlib import Path user_home = str(Path.home()) diff --git a/script/generate-mlperf-inference-user-conf/_cm.yaml b/script/generate-mlperf-inference-user-conf/_cm.yaml index 34960420a9..b172955407 100644 --- a/script/generate-mlperf-inference-user-conf/_cm.yaml +++ b/script/generate-mlperf-inference-user-conf/_cm.yaml @@ -27,6 +27,9 @@ default_env: CM_FAST_FACTOR: '5' CM_MLPERF_QUANTIZATION: off +docker: + real_run: False + # Map script inputs to environment variables input_mapping: count: CM_MLPERF_LOADGEN_QUERY_COUNT diff --git a/script/get-dataset-openimages-annotations/_cm.json b/script/get-dataset-openimages-annotations/_cm.json index 2cfa033e21..e77d44c0ce 100644 --- a/script/get-dataset-openimages-annotations/_cm.json +++ b/script/get-dataset-openimages-annotations/_cm.json @@ -17,13 +17,31 @@ "annotations" ], "uid": "47e2158ed24c44e9", + "prehook_deps": [ + { + "tags": "download-and-extract,_wget,_extract", + "env": { + "CM_DAE_FINAL_ENV_NAME": "CM_DATASET_ANNOTATIONS_FILE_PATH" + }, + "update_tags_from_env_with_prefix": { + "_url.": [ "CM_PACKAGE_URL" ] + }, + "force_cache": true, + "extra_cache_tags": "retinanet,get,dataset-openimages-annotations" + } + ], "variations": { "from.github": { "default": true, "env": { - "CM_WGET_URL": "https://github.com/mlcommons/inference/releases/download/v2.1/openimages-mlperf_annotations_2.1.json.zip" + "CM_DOWNLOAD_FILENAME": "openimages-mlperf_annotations_2.1.json.zip", + "CM_DOWNLOAD_CHECKSUM": "817fd8da3aeeb0575f1e2d2926b15e68", + "CM_PACKAGE_URL": "https://github.com/mlcommons/inference/releases/download/v2.1/openimages-mlperf_annotations_2.1.json.zip" }, "group": "download-source" } + }, + "print_env_at_the_end" : { + "CM_DATASET_ANNOTATIONS_FILE_PATH": "Path to OpenImages annotation file" } } diff --git a/script/get-dataset-openimages-annotations/customize.py b/script/get-dataset-openimages-annotations/customize.py index fa5f4b2630..a8954c67d6 100644 --- a/script/get-dataset-openimages-annotations/customize.py +++ b/script/get-dataset-openimages-annotations/customize.py @@ -6,9 +6,6 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - url = env['CM_WGET_URL'] - filename=os.path.basename(url) - env['CM_WGET_ZIP_FILE_NAME'] = filename return {'return':0} @@ -16,8 +13,8 @@ def preprocess(i): def postprocess(i): env = i['env'] - env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = os.path.join(os.getcwd(), 'openimages-mlperf.json') - env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join(os.getcwd()) + env['CM_DATASET_ANNOTATIONS_FILE_PATH'] = os.path.join(env['CM_DATASET_ANNOTATIONS_FILE_PATH'], 'openimages-mlperf.json') + env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.dirname(env['CM_DATASET_ANNOTATIONS_FILE_PATH']) env['CM_DATASET_OPENIMAGES_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] env['CM_DATASET_OPENIMAGES_ANNOTATIONS_DIR_PATH'] = env['CM_DATASET_ANNOTATIONS_DIR_PATH'] diff --git a/script/get-dataset-openimages-annotations/run.sh b/script/get-dataset-openimages-annotations/run.sh deleted file mode 100644 index 72c8557a4e..0000000000 --- a/script/get-dataset-openimages-annotations/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -cmd="wget -nc ${CM_WGET_URL} --no-check-certificate" -echo $cmd -eval $cmd -cmd="unzip ${CM_WGET_ZIP_FILE_NAME}" -echo $cmd -eval $cmd -test $? -eq 0 || exit 1 diff --git a/script/get-dataset-openimages-calibration/_cm.yaml b/script/get-dataset-openimages-calibration/_cm.yaml index 742a494d2f..b8bd73e12c 100644 --- a/script/get-dataset-openimages-calibration/_cm.yaml +++ b/script/get-dataset-openimages-calibration/_cm.yaml @@ -32,10 +32,12 @@ variations: default: true env: CM_MLPERF_OPENIMAGES_CALIBRATION_OPTION: one - CM_DOWNLOAD_CHECKSUM1: f09719174af3553119e2c621157773a6 + CM_DOWNLOAD_CHECKSUM: 5c3196ddcec4605c6a9fcf004d9615e6 adr: calibration-file-downloader: tags: _url.https://github.com/mlcommons/inference/raw/master/calibration/openimages/openimages_cal_images_list.txt + env: + CM_DOWNLOAD_CHECKSUM: 5c3196ddcec4605c6a9fcf004d9615e6 filter: default_variations: filter-size: filter_size.400 diff --git a/script/get-mlperf-inference-nvidia-scratch-space/_cm.json b/script/get-mlperf-inference-nvidia-scratch-space/_cm.json index 0ff47e4b8b..729a9d7844 100644 --- a/script/get-mlperf-inference-nvidia-scratch-space/_cm.json +++ b/script/get-mlperf-inference-nvidia-scratch-space/_cm.json @@ -36,10 +36,22 @@ }, "version.4_0": { "group": "version", - "default": true, "env": { "CM_NVIDIA_SCRATCH_SPACE_VERSION": "4_0" } + }, + "version.4_1": { + "group": "version", + "env": { + "CM_NVIDIA_SCRATCH_SPACE_VERSION": "4_1" + } + }, + "version.4_1-dev": { + "group": "version", + "default": true, + "env": { + "CM_NVIDIA_SCRATCH_SPACE_VERSION": "4_1-dev" + } } }, "versions": {}, diff --git a/script/get-mlperf-inference-results-dir/_cm.json b/script/get-mlperf-inference-results-dir/_cm.json index 843d83506c..81b599a3e9 100644 --- a/script/get-mlperf-inference-results-dir/_cm.json +++ b/script/get-mlperf-inference-results-dir/_cm.json @@ -31,6 +31,9 @@ ], "uid": "84f3c5aad5e1444b", "variations": { + "path.#": { + "CM_MLPERF_INFERENCE_RESULTS_DIR" : "#" + }, "version.#": { "group": "version", "env": { @@ -39,10 +42,16 @@ }, "version.4_1": { "group": "version", - "default": true, "env": { "CM_MLPERF_INFERENCE_RESULTS_VERSION": "4_1" } + }, + "version.4_1-dev": { + "group": "version", + "default": true, + "env": { + "CM_MLPERF_INFERENCE_RESULTS_VERSION": "4_1-dev" + } } }, "versions": {} diff --git a/script/get-mlperf-inference-submission-dir/_cm.json b/script/get-mlperf-inference-submission-dir/_cm.json index 5a39dc5b7c..59c95b7b39 100644 --- a/script/get-mlperf-inference-submission-dir/_cm.json +++ b/script/get-mlperf-inference-submission-dir/_cm.json @@ -37,10 +37,10 @@ }, "group": "version" }, - "version.4_1": { + "version.4_1-dev": { "default": true, "env": { - "CM_MLPERF_INFERENCE_SUBMISSION_VERSION": "4_1" + "CM_MLPERF_INFERENCE_SUBMISSION_VERSION": "4_1-dev" }, "group": "version" } diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index 9af1a4f997..7acbd4adf5 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -61,15 +61,17 @@ }, { "tags": "get,generic-python-lib,_package.nltk", - "names": + "names": [ "pip-package", "nltk" - ] + ], + "version_max": "3.8.1", + "version_max_usable": "3.8.1" }, { "tags": "get,generic-python-lib,_package.evaluate", - "names": + "names": [ "pip-package", "evaluate" @@ -77,7 +79,7 @@ }, { "tags": "get,generic-python-lib,_package.absl-py", - "names": + "names": [ "pip-package", "absl-py" diff --git a/script/run-docker-container/_cm.yaml b/script/run-docker-container/_cm.yaml index 8064f0dea2..d070bcad36 100644 --- a/script/run-docker-container/_cm.yaml +++ b/script/run-docker-container/_cm.yaml @@ -25,6 +25,7 @@ input_mapping: detached: CM_DOCKER_DETACHED_MODE device: CM_DOCKER_ADD_DEVICE docker_image_base: CM_DOCKER_IMAGE_BASE + keep_detached: CM_KEEP_DETACHED_CONTAINER docker_os: CM_DOCKER_OS docker_os_version: CM_DOCKER_OS_VERSION extra_run_args: CM_DOCKER_EXTRA_RUN_ARGS @@ -49,6 +50,9 @@ input_mapping: script_tags: CM_DOCKER_RUN_SCRIPT_TAGS shm_size: CM_DOCKER_SHM_SIZE +new_env_keys: + - 'CM_DOCKER_CONTAINER_ID' + prehook_deps: - names: - build-docker-image @@ -57,4 +61,6 @@ prehook_deps: - 'yes' CM_DOCKER_SKIP_BUILD: - 'yes' + CM_DOCKER_CONTAINER_ID: + - on tags: build,docker,image diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 1746280267..9158fde9a2 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -12,7 +12,7 @@ def preprocess(i): interactive = env.get('CM_DOCKER_INTERACTIVE_MODE','') - if interactive: + if str(interactive).lower() in ['yes', 'true', '1' ]: env['CM_DOCKER_DETACHED_MODE']='no' if 'CM_DOCKER_RUN_SCRIPT_TAGS' not in env: @@ -21,14 +21,6 @@ def preprocess(i): else: CM_RUN_CMD="cm run script --tags=" + env['CM_DOCKER_RUN_SCRIPT_TAGS'] + ' --quiet' - # Updating Docker info - update_docker_info(env) - - docker_image_repo = env['CM_DOCKER_IMAGE_REPO'] - docker_image_base = env['CM_DOCKER_IMAGE_BASE'] - docker_image_name = env['CM_DOCKER_IMAGE_NAME'] - docker_image_tag = env['CM_DOCKER_IMAGE_TAG'] - r = cm.access({'action':'search', 'automation':'script', 'tags': env['CM_DOCKER_RUN_SCRIPT_TAGS']}) @@ -40,37 +32,69 @@ def preprocess(i): env['CM_DOCKER_RUN_CMD'] = CM_RUN_CMD - DOCKER_CONTAINER = docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + # Updating Docker info + update_docker_info(env) - CMD = "docker images -q " + DOCKER_CONTAINER + docker_image_repo = env['CM_DOCKER_IMAGE_REPO'] + docker_image_base = env['CM_DOCKER_IMAGE_BASE'] + docker_image_name = env['CM_DOCKER_IMAGE_NAME'] + docker_image_tag = env['CM_DOCKER_IMAGE_TAG'] + + DOCKER_CONTAINER = docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + print ('') + print ('Checking existing Docker container:') + print ('') + CMD = f"""docker ps --filter "ancestor={DOCKER_CONTAINER}" """ if os_info['platform'] == 'windows': CMD += " 2> nul" else: CMD += " 2> /dev/null" - - print ('') - print ('Checking Docker images:') - print ('') print (' '+CMD) print ('') - + try: - docker_image = subprocess.check_output(CMD, shell=True).decode("utf-8") + docker_container = subprocess.check_output(CMD, shell=True).decode("utf-8") except Exception as e: return {'return':1, 'error':'Docker is either not installed or not started:\n{}'.format(e)} - recreate_image = env.get('CM_DOCKER_IMAGE_RECREATE', '') + output_split = docker_container.split("\n") + if len(output_split) > 1: #container exists + out = output_split[1].split(" ") + existing_container_id = out[0] + env['CM_DOCKER_CONTAINER_ID'] = existing_container_id + - if recreate_image != 'yes': - if docker_image: - print("Docker image exists with ID: " + docker_image) - env['CM_DOCKER_IMAGE_EXISTS'] = "yes" + else: + CMD = "docker images -q " + DOCKER_CONTAINER -# elif recreate_image == "yes": -# env['CM_DOCKER_IMAGE_RECREATE'] = "no" + if os_info['platform'] == 'windows': + CMD += " 2> nul" + else: + CMD += " 2> /dev/null" + print ('') + print ('Checking Docker images:') + print ('') + print (' '+CMD) + print ('') + try: + docker_image = subprocess.check_output(CMD, shell=True).decode("utf-8") + except Exception as e: + return {'return':1, 'error':'Docker is either not installed or not started:\n{}'.format(e)} + + recreate_image = env.get('CM_DOCKER_IMAGE_RECREATE', '') + + if recreate_image != 'yes': + if docker_image: + print("Docker image exists with ID: " + docker_image) + env['CM_DOCKER_IMAGE_EXISTS'] = "yes" + + # elif recreate_image == "yes": + # env['CM_DOCKER_IMAGE_RECREATE'] = "no" + + return {'return':0} def postprocess(i): @@ -164,14 +188,23 @@ def postprocess(i): run_opts += port_map_cmd_string # Currently have problem running Docker in detached mode on Windows: - detached = env.get('CM_DOCKER_DETACHED_MODE','') in ['yes', 'True', True] + detached = str(env.get('CM_DOCKER_DETACHED_MODE','')).lower() in ['yes', 'true', "1"] # if detached and os_info['platform'] != 'windows': if detached: if os_info['platform'] == 'windows': return {'return':1, 'error':'Currently we don\'t support running Docker containers in detached mode on Windows - TBD'} - CONTAINER="docker run -dt "+ run_opts + " --rm " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + " bash" - CMD = "ID=`" + CONTAINER + "` && docker exec $ID bash -c '" + run_cmd + "' && docker kill $ID >/dev/null" + existing_container_id = env.get('CM_DOCKER_CONTAINER_ID', '') + if existing_container_id: + CMD = f"ID={existing_container_id} && docker exec $ID bash -c '" + run_cmd + "'" + else: + CONTAINER="docker run -dt "+ run_opts + " --rm " + docker_image_repo + "/" + docker_image_name + ":" + docker_image_tag + " bash" + CMD = "ID=`" + CONTAINER + "` && docker exec $ID bash -c '" + run_cmd + "'" + + if False and str(env.get('CM_KEEP_DETACHED_CONTAINER', '')).lower() not in [ 'yes', "1", 'true' ]: + CMD += " && docker kill $ID >/dev/null" + + CMD += ' && echo "ID=$ID"' print ('=========================') print ("Container launch command:") @@ -185,6 +218,15 @@ def postprocess(i): print ('') docker_out = subprocess.check_output(CMD, shell=True).decode("utf-8") + lines = docker_out.split("\n") + + for line in lines: + print(f"line = {line}") + if line.startswith("ID="): + ID = line[3:] + print(f"My id = {ID}") + env['CM_DOCKER_CONTAINER_ID'] = ID + print(docker_out) else: @@ -243,8 +285,9 @@ def record_script(i): return {'return':0} def update_docker_info(env): + # Updating Docker info - docker_image_repo = env.get('CM_DOCKER_IMAGE_REPO', 'cknowledge') + docker_image_repo = env.get('CM_DOCKER_IMAGE_REPO', 'local') env['CM_DOCKER_IMAGE_REPO'] = docker_image_repo docker_image_base = env.get('CM_DOCKER_IMAGE_BASE') @@ -253,10 +296,14 @@ def update_docker_info(env): docker_image_base = env["CM_DOCKER_OS"]+":"+env["CM_DOCKER_OS_VERSION"] else: docker_image_base = "ubuntu:22.04" + env['CM_DOCKER_IMAGE_BASE'] = docker_image_base - docker_image_name = env.get('CM_DOCKER_IMAGE_NAME', 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','')) - env['CM_DOCKER_IMAGE_NAME'] = docker_image_name + if env.get('CM_DOCKER_IMAGE_NAME', '') != '': + docker_image_name = env['CM_DOCKER_IMAGE_NAME'] + else: + docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-') + env['CM_DOCKER_IMAGE_NAME'] = docker_image_name docker_image_tag_extra = env.get('CM_DOCKER_IMAGE_TAG_EXTRA', '-latest') diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 82466d011d..25bbde364a 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -271,12 +271,27 @@ variations: CM_MLPERF_INFERENCE_VERSION: '4.0' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.0_default group: benchmark-version + adr: + get-mlperf-inference-results-dir: + tags: _version.r4_0-dev + get-mlperf-inference-submission-dir: + tags: _version.r4_0-dev + mlperf-inference-nvidia-scratch-space: + tags: _version.r4_0-dev r4.1-dev: + default: true env: CM_MLPERF_INFERENCE_VERSION: '4.1-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.1-dev_default group: benchmark-version + adr: + get-mlperf-inference-results-dir: + tags: _version.r4_1-dev + get-mlperf-inference-submission-dir: + tags: _version.r4_1-dev + mlperf-inference-nvidia-scratch-space: + tags: _version.r4_1-dev r4.1: env: @@ -284,17 +299,18 @@ variations: CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.1_default adr: get-mlperf-inference-results-dir: - version: "r4.1" + tags: _version.r4_1 get-mlperf-inference-submission-dir: - version: "r4.1" + tags: _version.r4_1 mlperf-inference-nvidia-scratch-space: - version: "r4.1" + tags: _version.r4_1 group: benchmark-version short: add_deps_recursive: submission-checker: tags: _short-run + default: 'true' env: CM_MLPERF_SUBMISSION_GENERATION_STYLE: short @@ -327,9 +343,6 @@ variations: enable_if_env: CM_MLPERF_SKIP_SUBMISSION_GENERATION: - 'no' - - 'false' - - 'False' - - '0' tags: generate,mlperf,inference,submission versions: diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 7f566cf844..0722c016e1 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -100,6 +100,7 @@ def preprocess(i): if env.get('OUTPUT_BASE_DIR', '') == '': env['OUTPUT_BASE_DIR'] = env.get('CM_MLPERF_INFERENCE_RESULTS_DIR', os.getcwd()) + test_list = ["TEST01", "TEST05"] if env['CM_MODEL'] in ["resnet50"]: test_list.append("TEST04") @@ -131,6 +132,7 @@ def preprocess(i): add_deps = inp.get('add_deps', {}) ad = inp.get('ad', {}) adr = inp.get('adr', {}) + docker_it = inp.get('docker_it', '') adr_from_meta = i['run_script_input'].get('add_deps_recursive') for key in adr_from_meta: @@ -180,12 +182,12 @@ def preprocess(i): if str(env.get('CM_MLPERF_USE_DOCKER', '')).lower() in [ "1", "true", "yes"]: action = "docker" - del(env['OUTPUT_BASE_DIR']) + #del(env['OUTPUT_BASE_DIR']) state = {} docker_extra_input = {} - if env.get('CM_HW_NAME'): - del(env['CM_HW_NAME']) + #if env.get('CM_HW_NAME'): + # del(env['CM_HW_NAME']) for k in inp: if k.startswith("docker_"): @@ -216,19 +218,31 @@ def preprocess(i): for mode in env['CM_MLPERF_LOADGEN_MODES']: env['CM_MLPERF_LOADGEN_MODE'] = mode + env_copy = copy.deepcopy(env) print(f"\nRunning loadgen scenario: {scenario} and mode: {mode}") ii = {'action':action, 'automation':'script', 'tags': scenario_tags, 'quiet': 'true', - 'env': copy.deepcopy(env), 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': + 'env': env_copy, 'input': inp, 'state': state, 'add_deps': copy.deepcopy(add_deps), 'add_deps_recursive': copy.deepcopy(add_deps_recursive), 'ad': ad, 'adr': copy.deepcopy(adr), 'v': verbose, 'print_env': print_env, 'print_deps': print_deps, 'dump_version_info': dump_version_info} + if action == "docker": for k in docker_extra_input: ii[k] = docker_extra_input[k] r = cm.access(ii) if r['return'] > 0: return r - if action == "docker": - return {'return': 0} # We run commands interactively inside the docker container + if env_copy.get('CM_MLPERF_INFERENCE_FINAL_RESULTS_DIR', '') != '': + env['CM_MLPERF_INFERENCE_RESULTS_DIR_'] = env_copy['CM_MLPERF_INFERENCE_FINAL_RESULTS_DIR'] + else: + env['CM_MLPERF_INFERENCE_RESULTS_DIR_'] = os.path.join(env['OUTPUT_BASE_DIR'], f"{env['CM_MLPERF_RUN_STYLE']}_results") + + if action == "docker": + if str(docker_it).lower() not in ["no", "false", "0"]: + print(f"\nStop Running loadgen scenario: {scenario} and mode: {mode}") + return {'return': 0} # We run commands interactively inside the docker container + else: + container_id = env_copy['CM_DOCKER_CONTAINER_ID'] + env['CM_DOCKER_CONTAINER_ID'] = container_id if state.get('docker', {}): del(state['docker']) diff --git a/script/test-cm-core/src/script/test_docker.py b/script/test-cm-core/src/script/test_docker.py index 5a02c932ed..0ec418f214 100644 --- a/script/test-cm-core/src/script/test_docker.py +++ b/script/test-cm-core/src/script/test_docker.py @@ -8,14 +8,31 @@ 'tags': 'run,docker,container', 'add_deps_recursive': { 'compiler': {'tags': "gcc"} - }, + }, + 'docker_cm_repo': 'mlcommons@ck', 'image_name':'cm-script-app-image-classification-onnx-py', 'env': { 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', - 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', - 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04' + 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04', + 'CM_DOCKER_IMAGE_REPO': 'cknowledge' }, 'quiet': 'yes' }) +checks.check_return(r) +r = cm.access({'action':'run', + 'automation':'script', + 'tags': 'run,docker,container', + 'add_deps_recursive': { + 'compiler': {'tags': "gcc"} + }, + 'docker_cm_repo': 'mlcommons@ck', + 'image_name':'cm-script-app-image-classification-onnx-py', + 'env': { + 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', + 'CM_DOCKER_IMAGE_BASE': 'ubuntu:24.04', + 'CM_DOCKER_IMAGE_REPO': 'local' + }, + 'quiet': 'yes' + }) checks.check_return(r)