Add default config as json file (#272)

Contribute.md

@@ -17,11 +17,25 @@ required:
    ![Benchmarks Directory Structure](benchmarks_directory_structure.png)
 
 2. Next, in the leaf folder that was created in the previous step, you
-   will need to create a `model_init.py` file:
+   will need to create `config.json` and `model_init.py` files:
 
-   ![Add model init](add_model_init.png)
+   ![Add model init](add_model_init_and_config.png)
 
-   This file is used to initialize the best known configuration for the
+    The `config.json` file contains the best known KMP environment variable
+    settings to get optimal performance for the model. Below default settings are recommended for most of
+    the models in Model Zoo.
+
+    ```
+    {
+        "optimization_parameters": {
+            "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+            "KMP_BLOCKTIME": 1,
+            "KMP_SETTINGS": 1
+        }
+    }
+    ```
+
+   The `model_init.py` file is used to initialize the best known configuration for the
    model, and then start executing inference or training. When the
    [launch script](/docs/general/tensorflow/LaunchBenchmark.md) is run,
    it will look for the appropriate `model_init.py` file to use
@@ -33,7 +47,7 @@ required:
    [base model init class](/benchmarks/common/base_model_init.py) that
    includes functions for doing common tasks such as setting up the best
    known environment variables (like `KMP_BLOCKTIME`, `KMP_SETTINGS`,
-   `KMP_AFFINITY`, and `OMP_NUM_THREADS`), num intra threads, and num
+   `KMP_AFFINITY` by loading **config.json** and `OMP_NUM_THREADS`), num intra threads, and num
    inter threads. The `model_init.py` file also sets up the string that
    will ultimately be used to run inference or model training, which
    normally includes the use of `numactl` and sending all of the
@@ -93,7 +107,7 @@ Optional step:
   the original repository, then it can be added to the
   [models](/models) directory in the zoo repo. As with the first step
   in the previous section, the directory structure should be setup like:
-  `/models/<use case>/<framework>/<model name>/<mode>/<precision>`:
+  `/models/<use case>/<framework>/<model name>/<mode>/<precision>`.
 
   ![Models Directory Structure](models_directory_structure.png)
 

benchmarks/adversarial_networks/tensorflow/dcgan/inference/fp32/config.json

@@ -0,0 +1,8 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1,
+        "KMP_HW_SUBSET": "1T"
+    }
+}

benchmarks/adversarial_networks/tensorflow/dcgan/inference/fp32/model_init.py

@@ -37,8 +37,8 @@ def __init__(self, args, custom_args=[], platform_util=None):
         self.set_num_inter_intra_threads()
 
         # Set KMP env vars, if they haven't already been set
-        self.set_kmp_vars()
-        set_env_var("KMP_HW_SUBSET", "1T")
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         benchmark_script = os.path.join(
             self.args.intelai_models, args.mode, args.precision,

benchmarks/common/base_model_init.py

@@ -18,6 +18,7 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 
+import json
 import os
 
 
@@ -135,14 +136,28 @@ def set_num_inter_intra_threads(self, num_inter_threads=None, num_intra_threads=
             print("num_inter_threads: {}\nnum_intra_threads: {}".format(
                 self.args.num_inter_threads, self.args.num_intra_threads))
 
-    def set_kmp_vars(self, kmp_settings="1", kmp_blocktime="1", kmp_affinity="granularity=fine,verbose,compact,1,0"):
+    def set_kmp_vars(self, config_file_path, kmp_settings=None, kmp_blocktime=None, kmp_affinity=None):
         """
         Sets KMP_* environment variables to the specified value, if the environment variable has not already been set.
-        The default values for this function's args are the most common values that we have seen in the model zoo.
+        The default values in the json file are the best known settings for the model.
         """
+        if os.path.exists(config_file_path):
+            with open(config_file_path, 'r') as config:
+                config_object = json.load(config)
+
+            # First sets default from config file
+            for param in config_object.keys():
+                for env in config_object[param].keys():
+                    set_env_var(env, config_object[param][env])
+
+        else:
+            print("Warning: File {} does not exist and \
+            cannot be used to set KMP environment variables".format(config_file_path))
+
+        # Override user provided envs
         if kmp_settings:
-            set_env_var("KMP_SETTINGS", kmp_settings)
+            set_env_var("KMP_SETTINGS", kmp_settings, overwrite_existing=True)
         if kmp_blocktime:
-            set_env_var("KMP_BLOCKTIME", kmp_blocktime)
+            set_env_var("KMP_BLOCKTIME", kmp_blocktime, overwrite_existing=True)
         if kmp_affinity:
-            set_env_var("KMP_AFFINITY", kmp_affinity)
+            set_env_var("KMP_AFFINITY", kmp_affinity, overwrite_existing=True)

benchmarks/content_creation/tensorflow/draw/inference/fp32/config.json

@@ -0,0 +1,8 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1,
+        "KMP_HW_SUBSET": "1T"
+    }
+}

benchmarks/content_creation/tensorflow/draw/inference/fp32/model_init.py

@@ -22,7 +22,6 @@
 import os
 import sys
 from common.base_model_init import BaseModelInitializer
-from common.base_model_init import set_env_var
 
 
 class ModelInitializer(BaseModelInitializer):
@@ -32,8 +31,8 @@ def __init__(self, args, custom_args=[], platform_util=None):
         super(ModelInitializer, self).__init__(args, custom_args, platform_util)
 
         # Set KMP env vars, if they haven't already been set
-        self.set_kmp_vars()
-        set_env_var("KMP_HW_SUBSET", "1T")
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         if self.args.accuracy_only:
             print("Accuracy testing for DRAW inference is not supported yet.")

benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/fp32/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/face_detection_and_alignment/tensorflow/facenet/inference/fp32/model_init.py

@@ -34,7 +34,8 @@ def __init__(self, args, custom_args=[], platform_util=None):
             self.python_exe + " "
 
         # Set KMP env vars, if they haven't already been set
-        self.set_kmp_vars()
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         pairs_file = os.path.join(self.args.model_source_dir,
                                   "data/pairs.txt")

benchmarks/face_detection_and_alignment/tensorflow/mtcc/inference/fp32/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/face_detection_and_alignment/tensorflow/mtcc/inference/fp32/model_init.py

@@ -33,7 +33,8 @@ def __init__(self, args, custom_args, platform_util=None):
         self.set_num_inter_intra_threads()
 
         # Set KMP env vars, if they haven't already been set
-        self.set_kmp_vars()
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         set_env_var("OMP_NUM_THREADS", self.args.num_intra_threads)
 

benchmarks/image_recognition/tensorflow/inception_resnet_v2/inference/fp32/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/inception_resnet_v2/inference/fp32/model_init.py

@@ -32,7 +32,8 @@ def __init__(self, args, custom_args=[], platform_util=None):
         self.cmd = self.get_numactl_command(self.args.socket_id) + self.python_exe + " "
 
         # Set KMP env vars, if they haven't already been set
-        self.set_kmp_vars()
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         # use default batch size if -1
         if self.args.batch_size == -1:

benchmarks/image_recognition/tensorflow/inception_resnet_v2/inference/int8/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/inception_resnet_v2/inference/int8/model_init.py

@@ -31,7 +31,11 @@ class ModelInitializer(BaseModelInitializer):
 
     def __init__(self, args, custom_args=[], platform_util=None):
         super(ModelInitializer, self).__init__(args, custom_args, platform_util)
-        self.set_kmp_vars()
+
+        # Set KMP env vars, if they haven't already been set
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
+
         self.cmd = self.get_numactl_command(self.args.socket_id) + "{} ".format(self.python_exe)
 
         # use default batch size if -1

benchmarks/image_recognition/tensorflow/inceptionv3/inference/fp32/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/inceptionv3/inference/fp32/model_init.py

@@ -60,8 +60,10 @@ def __init__(self, args, custom_args=[], platform_util=None):
 
         self.args = arg_parser.parse_args(self.custom_args, namespace=self.args)
 
-        # Use default KMP variable values, but override the default KMP_BLOCKTIME value
-        self.set_kmp_vars(kmp_blocktime=str(self.args.kmp_blocktime))
+        # Set KMP env vars, if they haven't already been set, but override the default KMP_BLOCKTIME value
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path, kmp_blocktime=str(self.args.kmp_blocktime))
+
         set_env_var("OMP_NUM_THREADS", self.args.num_intra_threads)
 
         benchmark_script = os.path.join(

benchmarks/image_recognition/tensorflow/inceptionv3/inference/int8/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/inceptionv3/inference/int8/model_init.py

@@ -60,8 +60,9 @@ def parse_args(self):
 
         self.args = parser.parse_args(self.custom_args,
                                       namespace=self.args)
-        # Use default KMP variable values, but override the default KMP_BLOCKTIME value
-        self.set_kmp_vars(kmp_blocktime=str(self.args.kmp_blocktime))
+        # Set KMP env vars, if they haven't already been set, but override the default KMP_BLOCKTIME value
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path, kmp_blocktime=str(self.args.kmp_blocktime))
 
     def run_benchmark(self):
         benchmark_script = os.path.join(self.args.intelai_models,

benchmarks/image_recognition/tensorflow/inceptionv4/inference/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 0,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/inceptionv4/inference/inceptionv4_model_init.py

@@ -38,7 +38,11 @@ def __init__(self, args, custom_args=[], platform_util=None):
         # Environment variables
         set_env_var("OMP_NUM_THREADS", platform_util.num_cores_per_socket
                     if self.args.num_cores == -1 else self.args.num_cores)
-        self.set_kmp_vars(kmp_blocktime="0")
+
+        # Set KMP env vars, if they haven't already been set
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
+
         self.set_num_inter_intra_threads(num_inter_threads=platform_util.num_threads_per_core,
                                          num_intra_threads=platform_util.num_cores_per_socket)
 

benchmarks/image_recognition/tensorflow/mobilenet_v1/inference/fp32/config.json

@@ -0,0 +1,6 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1
+    }
+}

benchmarks/image_recognition/tensorflow/mobilenet_v1/inference/fp32/model_init.py

@@ -33,8 +33,9 @@ def __init__(self, args, custom_args=[], platform_util=None):
         if self.args.batch_size == -1:
             self.args.batch_size = 128
 
-        # Set KMP env vars (except KMP_SETTINGS is not set)
-        self.set_kmp_vars(kmp_settings=None)
+        # Set KMP env vars, if they haven't already been set
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         # set num_inter_threads and num_intra_threads (override inter threads to 2)
         self.set_num_inter_intra_threads(num_inter_threads=2)

benchmarks/image_recognition/tensorflow/mobilenet_v1/inference/int8/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/mobilenet_v1/inference/int8/model_init.py

@@ -37,7 +37,8 @@ def __init__(self, args, custom_args=[], platform_util=None):
         self.cmd = self.get_numactl_command(self.args.socket_id) + "python "
 
         # Set KMP env vars, if they haven't already been set
-        self.set_kmp_vars()
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
         # Set the num_inter_threads and num_intra_threads
         self.set_num_inter_intra_threads()

benchmarks/image_recognition/tensorflow/resnet101/inference/fp32/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/resnet101/inference/fp32/model_init.py

@@ -60,8 +60,10 @@ def __init__(self, args, custom_args=[], platform_util=None):
 
         self.args = arg_parser.parse_args(self.custom_args, namespace=self.args)
 
-        # Use default KMP variable values, but override the default KMP_BLOCKTIME value
-        self.set_kmp_vars(kmp_blocktime=str(self.args.kmp_blocktime))
+        # Set KMP env vars, if they haven't already been set, but override the default KMP_BLOCKTIME value
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path, kmp_blocktime=str(self.args.kmp_blocktime))
+
         set_env_var("OMP_NUM_THREADS", self.args.num_intra_threads)
 
         benchmark_script = os.path.join(

benchmarks/image_recognition/tensorflow/resnet101/inference/int8/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 0,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/resnet101/inference/int8/model_init.py

@@ -41,8 +41,9 @@ def __init__(self, args, custom_args=[], platform_util=None):
         set_env_var("OMP_NUM_THREADS",
                     platform_util.num_cores_per_socket if args.num_cores == -1 else args.num_cores)
 
-        # Set KMP env vars, but override default KMP_BLOCKTIME value
-        self.set_kmp_vars(kmp_blocktime="0")
+        # Set KMP env vars, if they haven't already been set
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path)
 
     def parse_args(self):
         parser = argparse.ArgumentParser()

benchmarks/image_recognition/tensorflow/resnet50/inference/fp32/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/resnet50/inference/fp32/model_init.py

@@ -61,8 +61,10 @@ def __init__(self, args, custom_args=[], platform_util=None):
 
         self.args = arg_parser.parse_args(self.custom_args, namespace=self.args)
 
-        # Use default KMP variable values, but override the default KMP_BLOCKTIME value
-        self.set_kmp_vars(kmp_blocktime=str(self.args.kmp_blocktime))
+        # Set KMP env vars, if they haven't already been set, but override the default KMP_BLOCKTIME value
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path, kmp_blocktime=str(self.args.kmp_blocktime))
+
         set_env_var("OMP_NUM_THREADS", self.args.num_intra_threads)
 
         benchmark_script = os.path.join(

benchmarks/image_recognition/tensorflow/resnet50/inference/int8/config.json

@@ -0,0 +1,7 @@
+{
+    "optimization_parameters": {
+        "KMP_AFFINITY": "granularity=fine,verbose,compact,1,0",
+        "KMP_BLOCKTIME": 1,
+        "KMP_SETTINGS": 1
+    }
+}

benchmarks/image_recognition/tensorflow/resnet50/inference/int8/model_init.py

@@ -65,8 +65,9 @@ def parse_args(self):
 
         self.args = parser.parse_args(self.custom_args,
                                       namespace=self.args)
-        # Use default KMP variable values, but override the default KMP_BLOCKTIME value
-        self.set_kmp_vars(kmp_blocktime=str(self.args.kmp_blocktime))
+        # Set KMP env vars, if they haven't already been set, but override the default KMP_BLOCKTIME value
+        config_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config.json")
+        self.set_kmp_vars(config_file_path, kmp_blocktime=str(self.args.kmp_blocktime))
 
     def run_benchmark_or_accuracy(self):
         cmd = os.path.join(