MLPerf Inference v4.0 release.

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+*.pyc
+*~

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022-2024 Krai Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,20 @@
+# axs2qaic
+
+Automated [KRAI X](http://github.com/krai/axs) workflows for reproducing MLPerf
+Inference submissions on systems equipped with [Qualcomm Cloud AI
+100](https://www.qualcomm.com/products/technology/processors/cloud-artificial-intelligence)
+accelerators.
+
+To import this repository and its dependencies into your **work_collection**, run:
+```
+axs byquery git_repo,collection,repo_name=axs2qaic
+```
+
+## License
+
+Unless explicitly stated otherwise, the software in this repository is provided
+under the permissive [MIT license](LICENSE.txt).
+
+## Contact
+
+Please contact [email protected] for any queries.

base_qaic_config/code_axs.py

@@ -0,0 +1,107 @@
+import json
+import os.path
+
+CARD_TYPE = ["std", "pro", "edge", "lite"]
+SYSTEM_TYPES = ["dc", "edge"]
+CHANGABLE_FIELDS = {
+    "num_device": -3,
+    "card": -2,
+    "system_type": -1
+}
+
+
+def _set_targets_field_and_value(org_name, target):
+    org_set = set(org_name.split("_")[-3:])
+    target_set = set(target.split("_")[-3:])
+    common_field = org_set.intersection(target_set)
+    assert len(common_field) ==  len(CHANGABLE_FIELDS)-1 , f"\n\n\nERROR: Cannot change more than 1 field at a time. \n\n\n"
+
+    target_value = next(iter(target_set-org_set))
+    if target_value in CARD_TYPE: 
+        target_field = "card"
+    elif target_value in SYSTEM_TYPES:
+        target_field = "system_type"
+    else:
+        target_field = "num_device"
+        target_value = int(target_value.replace('q', ''))
+
+    print(f"Setting target_field={target_field} and target_value={target_value}...")
+    return target_field, target_value
+
+
+
+def gen_config(target=None, target_field=None, target_value=None, preview=False, __entry__=None):
+    """To generate new entries based on existing config. Can only generate config entry based on another config in the same collection.
+
+Usage examples :
+        axs byname compile_model_bert-99_q1_pro_edge , gen_config --target=q1_pro_dc --preview+ # Take a look
+        axs byname compile_model_bert-99_q1_pro_edge , gen_config --target_field=system_type --target_val=dc --preview+ # Take a look
+        axs byname compile_model_bert-99_q1_pro_edge , gen_config --target=q1_pro_dc # Actually generate it
+        axs byname compile_model_bert-99_q1_pro_edge , gen_config --target_field=system_type --target_val=dc --preview+ # Actually generate it
+
+    """
+
+    assert target or (target_field and target_value), f"\n\n\nERROR: Please set either target or (target_field and target_vale). Get helps with: \n\naxs byname base_qaic_config , help gen_config \n\n\n"
+
+    target_container = __entry__.get_container()
+    ak = __entry__.get_kernel()
+    org_data = __entry__.own_data()
+    org_num_device = __entry__.get("num_device")
+    org_name = __entry__.get_name()
+
+    if target:
+        target_field, target_value = _set_targets_field_and_value(org_name, target)
+
+    assert target_field in CHANGABLE_FIELDS.keys() , f"\n\n\nERROR: This func does not support varying {target_field}, try one of these [{CHANGABLE_FIELDS.keys()}] instead.\n\n\n"
+    if target_field == "num_device":
+        assert target_value != 0 , f"\n\n\nERROR: There is no such thing as q0 entry! \n\n\n"
+    if target_field == "card":
+        assert target_value in CARD_TYPE, f"\n\n\nERROR: {target_value}  is not one of the supported. Try these instead [f{CARD_TYPE}].\n\n\n"
+    if target_field == "system_type":
+        assert target_value in SYSTEM_TYPES, f"\n\n\nERROR: {target_value}  is not one of the supported. Try these instead [f{SYSTEM_TYPES}].\n\n\n"
+
+    org_name_list = org_name.split("_")
+    stuff_to_delete = org_name_list[CHANGABLE_FIELDS[target_field]]
+    new_field = "q{}".format(target_value) if target_field == "num_device" else str(target_value)
+    stuff_to_delete_idx = org_name_list.index(stuff_to_delete)
+    org_name_list[stuff_to_delete_idx] = new_field
+    new_name = "_".join(org_name_list)
+
+    print(f"\n\nGenerating {new_name} based on {org_name} ...\n\n")
+
+    current_entries = target_container.get("contained_entries")
+    if new_name in current_entries:
+        print(f"\nEntry you try to create {new_name} already exist. Mission abort! \n")
+        return
+
+    output_entry = ak.fresh_entry(name=new_name)
+    output_entry.own_data(org_data)
+
+    output_entry.pluck(target_field)
+    new_value = int(target_value) if target_field == "num_device" else str(target_value)
+    output_entry.plant(target_field, new_value)
+
+    if target_field == "num_device":
+        assert target_value != 1 , f"\n\n\nERROR:Cannot improvised from {org_name} to {new_name}. Please generate q1 entry yourself.\n\n\n" # (q4 to q1) not supported
+        parents = output_entry.get("_parent_entries")
+        if org_num_device == 1:
+            # (q1 to q4)
+            parents[0] = ak.byname(org_name)
+            output_entry.pluck("_parent_entries")
+            output_entry.plant("_parent_entries", parents)
+
+            cur_dir = os.path.abspath(os.path.dirname(__file__))
+            file_path = os.path.join(cur_dir, "non_q1_parent.json")
+            with open(file_path) as f:
+                new_parent = json.load(f)
+            RED = '\033[31m'
+            END_COLOR = '\033[0m'
+            print(RED + f"\nWARNING: Please update the parents to look like this before committing\n\n{new_parent}\n" + END_COLOR)
+        #else:
+            # (q2 to q4) or (q4 to q2) # suported
+
+    if preview:
+        print(f"\nThis is how it will look like!\n")
+        return output_entry
+
+    return output_entry.attach(target_container).save()

base_qaic_config/data_axs.json

@@ -0,0 +1,3 @@
+{
+
+}

base_qaic_config/gen_q1_config.py

@@ -0,0 +1,169 @@
+import os
+import json
+import copy
+
+"""
+Generate all missing q1 config with this template. Must be excute in this directory.
+
+Usage example:
+    elim@velociti:~/work_collection/axs2qaic-dev/base_qaic_config$ python3 gen_q1_config.py
+
+"""
+
+ROOT_DIR = os.path.abspath(os.curdir)
+
+benchmarks= ["bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet"]
+device_nums= ["q1"]
+card_types= ["std", "pro"]
+system_types= ["dc", "edge"]
+config_targets=["compiletime_profile", "compiletime_model", "runtime"]
+
+config_target_filename_mapping= {
+    "compiletime_profile": "compile_profile",
+    "compiletime_model": "compile_model",
+    "runtime": "run"
+}
+
+config_target_tag_mapping= {
+    "compiletime_profile": "compile_profile",
+    "compiletime_model": "compile_model",
+    "runtime": "runtime"
+}
+
+template_json = {
+    "_parent_entries": [ 
+        [ "^", "byname", "base_qaic_config" ], 
+        [ "^", "byquery", [ "^^", "substitute", "model_config,device=#{device}#,model=#{model}#" ],{},["device", "model"] ],
+        [ "^", "byquery", [ "^^", "substitute", "description,device=#{device}#,card=#{card}#" ],{},["device", "card"] ]
+    ],
+    "_debug_path": [ "^^", "get_path"],
+
+    "tags": [ "device_model_config" ],
+    "config_target" : "runtime",
+    "device": "qaic",
+    "card": "pro",
+    "system_type": "edge",
+    "model": "bert-99",
+    "num_device": 1,
+
+    "SingleStream": {
+        "ERROR": [ "^^", "substitute", "This config is empty at #{_debug_path}#"]
+    },
+    "Offline": {
+        "ERROR": [ "^^", "substitute", "This config is empty at #{_debug_path}#"]
+    },
+    "Server": {
+        "ERROR": [ "^^", "substitute", "This config is empty at #{_debug_path}#"]
+    },
+    "MultiStream": {
+        "ERROR": [ "^^", "substitute", "This config is empty at #{_debug_path}#"]
+    }
+}
+
+template_json_no_error = {
+    "_parent_entries": [ 
+        [ "^", "byname", "base_qaic_config" ], 
+        [ "^", "byquery", [ "^^", "substitute", "model_config,device=#{device}#,model=#{model}#" ],{},["device", "model"] ],
+        [ "^", "byquery", [ "^^", "substitute", "description,device=#{device}#,card=#{card}#" ],{},["device", "card"] ]
+    ],
+    "_debug_path": [ "^^", "get_path"],
+
+    "tags": [ "device_model_config" ],
+    "config_target" : "runtime",
+    "device": "qaic",
+    "card": "pro",
+    "system_type": "edge",
+    "model": "bert-99",
+    "num_device": 1,
+
+    "SingleStream": {},
+
+    "Offline": {},
+
+    "Server": {},
+
+    "MultiStream": {}
+}
+
+def gen_must_exist_q1_config():
+    for config_target in config_targets:
+        for benchmark in benchmarks:
+            for device_num in device_nums:
+                for card_type in card_types:
+                    for system_type in system_types:
+
+                        path = f'{ROOT_DIR}/../config_for_model_and_device_{config_target}_collection/{config_target_filename_mapping[config_target]}_{benchmark}_{device_num}_{card_type}_{system_type}/data_axs.json'
+                        collection_path = os.path.join(os.path.dirname(os.path.dirname(path)),"data_axs.json")
+
+                        if config_target=="compiletime_profile" and benchmark=="bert-99.9":
+                            if os.path.exists(path):
+                                print("This shouldnt exist! Please delete entry manually.", path)
+                            continue
+
+                        if system_type=="edge" and benchmark=="bert-99.9":
+                            if os.path.exists(path):
+                                print("This shouldnt exist! Please delete entry manually.", path)
+                            continue
+
+                        assert os.path.exists(collection_path), f"Check ROOT_DIR is correct, please execute the script within the base_qaic_config folder."
+
+                        # Update exisiting broken q1 config
+                        if os.path.exists(path) and config_target == "compiletime_model":
+                            with open(path, "r") as f:
+                                cur_config = json.load(f)
+
+                            require_update = False
+                            for scenario in ["SingleStream", "Offline", "Server", "MultiStream"]:
+                                if scenario not in cur_config:
+                                    cur_config[scenario] = template_json[scenario]
+                                    require_update = True
+
+                            if require_update:
+                                json_object = json.dumps(cur_config, indent=4)
+                                print(json_object)
+                                with open(path, "w") as f:
+                                    f.write(json_object)
+
+                        # Create q1 configs
+                        if not os.path.exists(path):
+                            print(path)
+                            entry_path = os.path.dirname(path)
+                            entry_name = os.path.basename(entry_path)
+                            # print(entry_path)
+
+                            try:
+                                os.makedirs(entry_path)
+                            except:
+                                print("Folder already exists.")
+
+                            if config_target == "compiletime_model":
+                                this_json = copy.deepcopy(template_json)
+                            else:
+                                this_json = copy.deepcopy(template_json_no_error)
+
+                            this_json["config_target"] = config_target_tag_mapping[config_target]
+                            this_json["card"] = card_type
+                            this_json["system_type"] = system_type
+                            this_json["model"] = benchmark
+                            this_json["num_device"] = 1   
+                            json_object = json.dumps(this_json, indent=4)
+                            print(json_object)
+
+                            with open(path, "w") as f:
+                                f.write(json_object)
+
+
+                            print(collection_path)
+                            with open(collection_path, "r") as f:
+                                controller = json.load(f)
+
+                            exisiting_entries = controller["contained_entries"]
+                            exisiting_entries[entry_name] = entry_name
+                            controller["contained_entries"] = dict(sorted(exisiting_entries.items())) 
+                            json_object = json.dumps(controller, indent=4)
+                            print(json_object)
+
+                            with open(collection_path, "w") as f:
+                                f.write(json_object)
+
+gen_must_exist_q1_config()

base_qaic_config/non_q1_parent.json

@@ -0,0 +1,7 @@
+{
+    "_parent_entries":[
+        [ "^", "byquery", [ "^^", "substitute", "device_model_config,config_target=#{config_target}#,device=#{device}#,card=#{card}#,system_type=#{system_type}#,model=#{model}#,num_device=1"],{},["config_target", "device","card","system_type","model"]],
+        [ "^", "byquery", [ "^^", "substitute", "model_config,device=#{device}#,model=#{model}#"],{},["device","model"]] ,
+        [ "^", "byquery", [ "^^", "substitute", "description,device=#{device}#,card=#{card}#"],{},["device","card"] ] 
+    ]
+}

base_qaic_config/q1_parent.json

@@ -0,0 +1,7 @@
+{
+    "_parent_entries": [ 
+        [ "^", "byname", "base_qaic_config" ], 
+        [ "^", "byquery", [ "^^", "substitute", "model_config,device=#{device}#,model=#{model}#" ],{},["device", "model"] ],
+        [ "^", "byquery", [ "^^", "substitute", "description,device=#{device}#,card=#{card}#" ],{},["device", "card"] ]
+    ]
+}