inf1-neuron-sdk-resnet50.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AWS Inferentia inference on Amazon EC2 Inf1 instance\n",
    "This example demonstrates AWS Inferentia inference with TensorFlow and AWS Neuron SDK compiler and runtime\n",
    "\n",
    "This example was tested on Amazon EC2 `inf1.xlarge` the following AWS Deep Learning AMI: \n",
    "`Deep Learning AMI (Ubuntu 18.04) Version 35.0`\n",
    "\n",
    "Run this notebook using the following conda environment:\n",
    "`aws_neuron_tensorflow_p36`\n",
    "\n",
    "Prepare your imagenet validation TFRecord files using the following helper script:\n",
    "https://github.com/tensorflow/models/blob/archive/research/inception/inception/data/download_and_preprocess_imagenet.sh\n",
    "\n",
    "Save it to `/home/ubuntu/datasets/` or update the dataset location in the `get_dataset()` function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install matplotlib pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/opt/aws/neuron/bin/neuron-cli reset\n",
    "import os\n",
    "import time\n",
    "import shutil\n",
    "import json\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import tensorflow.neuron as tfn\n",
    "import tensorflow.compat.v1.keras as keras\n",
    "from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from concurrent import futures\n",
    "from itertools import compress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/tensorflow/tensorflow/issues/29931\n",
    "temp = tf.zeros([8, 224, 224, 3])\n",
    "_ = tf.keras.applications.resnet50.preprocess_input(temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resnet50 FP32 saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/aws_neuron_tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From <ipython-input-4-cc6f128953d9>:10: simple_save (from tensorflow.python.saved_model.simple_save) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.simple_save.\n",
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/aws_neuron_tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: resnet50_saved_model/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "# Export SavedModel\n",
    "saved_model_dir = 'resnet50_saved_model'\n",
    "shutil.rmtree(saved_model_dir, ignore_errors=True)\n",
    "\n",
    "keras.backend.set_learning_phase(0)\n",
    "model = ResNet50(weights='imagenet')\n",
    "tf.saved_model.simple_save(session = keras.backend.get_session(),\n",
    "                           export_dir = saved_model_dir,\n",
    "                           inputs = {'input_1:0': model.inputs[0]},\n",
    "                           outputs = {'probs/Softmax:0': model.outputs[0]})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compile models with different batch sizes and cores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compile_inf1_model(saved_model_dir, inf1_model_dir, batch_size=1, num_cores=1, use_static_weights=False):\n",
    "    print(f'-----------batch size: {batch_size}, num cores: {num_cores}----------')\n",
    "    print('Compiling...')\n",
    "    \n",
    "    compiled_model_dir = f'resnet50_batch_{batch_size}_inf1_cores_{num_cores}'\n",
    "    inf1_compiled_model_dir = os.path.join(inf1_model_dir, compiled_model_dir)\n",
    "    shutil.rmtree(inf1_compiled_model_dir, ignore_errors=True)\n",
    "\n",
    "    example_input = np.zeros([batch_size,224,224,3], dtype='float32')\n",
    "\n",
    "    compiler_args = ['--verbose','1', '--num-neuroncores', str(num_cores)]\n",
    "    if use_static_weights:\n",
    "        compiler_args.append('--static-weights')\n",
    "    \n",
    "    start_time = time.time()\n",
    "    compiled_res = tfn.saved_model.compile(model_dir = saved_model_dir,\n",
    "                            model_feed_dict={'input_1:0': example_input},\n",
    "                            new_model_dir = inf1_compiled_model_dir,\n",
    "                            dynamic_batch_size=True,\n",
    "                            compiler_workdir=f'./compiler-workdir/{inf1_compiled_model_dir}',\n",
    "                            compiler_args = compiler_args)\n",
    "    print(f'Compile time: {time.time() - start_time}')\n",
    "    \n",
    "    compile_success = False\n",
    "    perc_on_inf = compiled_res['OnNeuronRatio'] * 100\n",
    "    if perc_on_inf > 50:\n",
    "        compile_success = True\n",
    "            \n",
    "    print(inf1_compiled_model_dir)\n",
    "    print(compiled_res)\n",
    "    print('----------- Done! ----------- \\n')\n",
    "    \n",
    "    return compile_success"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use `tf.data` to read ImageNet validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deserialize_image_record(record):\n",
    "    feature_map = {'image/encoded': tf.io.FixedLenFeature([], tf.string, ''),\n",
    "                  'image/class/label': tf.io.FixedLenFeature([1], tf.int64, -1),\n",
    "                  'image/class/text': tf.io.FixedLenFeature([], tf.string, '')}\n",
    "    obj = tf.io.parse_single_example(serialized=record, features=feature_map)\n",
    "    imgdata = obj['image/encoded']\n",
    "    label = tf.cast(obj['image/class/label'], tf.int32)   \n",
    "    label_text = tf.cast(obj['image/class/text'], tf.string)   \n",
    "    return imgdata, label, label_text\n",
    "\n",
    "def val_preprocessing(record):\n",
    "    imgdata, label, label_text = deserialize_image_record(record)\n",
    "    label -= 1\n",
    "    image = tf.io.decode_jpeg(imgdata, channels=3, \n",
    "                              fancy_upscaling=False, \n",
    "                              dct_method='INTEGER_FAST')\n",
    "\n",
    "    shape = tf.shape(image)\n",
    "    height = tf.cast(shape[0], tf.float32)\n",
    "    width = tf.cast(shape[1], tf.float32)\n",
    "    side = tf.cast(tf.convert_to_tensor(256, dtype=tf.int32), tf.float32)\n",
    "\n",
    "    scale = tf.cond(tf.greater(height, width),\n",
    "                  lambda: side / width,\n",
    "                  lambda: side / height)\n",
    "    \n",
    "    new_height = tf.cast(tf.math.rint(height * scale), tf.int32)\n",
    "    new_width = tf.cast(tf.math.rint(width * scale), tf.int32)\n",
    "    \n",
    "    image = tf.image.resize(image, [new_height, new_width], method='bicubic')\n",
    "    image = tf.image.resize_with_crop_or_pad(image, 224, 224)\n",
    "    \n",
    "    image = tf.keras.applications.resnet50.preprocess_input(image)\n",
    "    \n",
    "    return image, label, label_text\n",
    "\n",
    "def get_dataset(batch_size, use_cache=False):\n",
    "    data_dir = '/home/ubuntu/datasets/*'\n",
    "    files = tf.io.gfile.glob(os.path.join(data_dir))\n",
    "    dataset = tf.data.TFRecordDataset(files)\n",
    "    \n",
    "    dataset = dataset.map(map_func=val_preprocessing, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "    dataset = dataset.batch(batch_size=batch_size)\n",
    "    dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n",
    "    dataset = dataset.repeat(count=1)\n",
    "    \n",
    "    if use_cache:\n",
    "        shutil.rmtree('tfdatacache', ignore_errors=True)\n",
    "        os.mkdir('tfdatacache')\n",
    "        dataset = dataset.cache(f'./tfdatacache/imagenet_val')\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single AWS Inferentia chip execution\n",
    "* Single core compiled models with automatic data parallel model upto 4 cores\n",
    "* Multi-core compiled models for pipeline execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def inf1_predict_benchmark_single_threaded(neuron_saved_model_name, batch_size, user_batch_size, num_cores, use_cache=False, warm_up=10):\n",
    "    print(f'Running model {neuron_saved_model_name}, user_batch_size: {user_batch_size}\\n')\n",
    "\n",
    "    model_inf1 = tf.contrib.predictor.from_saved_model(neuron_saved_model_name)\n",
    "\n",
    "    iter_times = []\n",
    "    pred_labels = []\n",
    "    actual_labels = []\n",
    "    display_threshold = 0\n",
    "    warm_up = 10\n",
    "\n",
    "    ds = get_dataset(user_batch_size, use_cache)\n",
    "\n",
    "    ds_iter = ds.make_initializable_iterator()\n",
    "    ds_next = ds_iter.get_next()\n",
    "    ds_init_op = ds_iter.initializer\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        if use_cache:\n",
    "            sess.run(ds_init_op)\n",
    "            print('\\nCaching dataset ...')\n",
    "            start_time = time.time()\n",
    "            try:\n",
    "                while True:\n",
    "                    (validation_ds,label,_) = sess.run(ds_next)\n",
    "            except tf.errors.OutOfRangeError:\n",
    "                pass\n",
    "            print(f'Caching finished: {time.time()-start_time} sec')  \n",
    "\n",
    "        try:\n",
    "            sess.run(ds_init_op)\n",
    "            counter = 0\n",
    "            \n",
    "            display_every = 5000\n",
    "            display_threshold = display_every\n",
    "            \n",
    "            ipname = list(model_inf1.feed_tensors.keys())[0]\n",
    "            resname = list(model_inf1.fetch_tensors.keys())[0]\n",
    "            \n",
    "            walltime_start = time.time()\n",
    "\n",
    "            while True:\n",
    "                (validation_ds,batch_labels,_) = sess.run(ds_next)\n",
    "\n",
    "                model_feed_dict={ipname: validation_ds}\n",
    "\n",
    "                if counter == 0:\n",
    "                    for i in range(warm_up):\n",
    "                        _ = model_inf1(model_feed_dict);                    \n",
    "\n",
    "                start_time = time.time()\n",
    "                inf1_results = model_inf1(model_feed_dict);\n",
    "                iter_times.append(time.time() - start_time)\n",
    "                \n",
    "                actual_labels.extend(label for label_list in batch_labels for label in label_list)\n",
    "                pred_labels.extend(list(np.argmax(inf1_results[resname], axis=1)))\n",
    "\n",
    "                if counter*user_batch_size >= display_threshold:\n",
    "                    print(f'Images {counter*user_batch_size}/50000. Average i/s {np.mean(user_batch_size/np.array(iter_times[-display_every:]))}')\n",
    "                    display_threshold+=display_every\n",
    "\n",
    "                counter+=1\n",
    "\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            pass\n",
    "        \n",
    "    acc_inf1 = np.sum(np.array(actual_labels) == np.array(pred_labels))/len(actual_labels)\n",
    "    iter_times = np.array(iter_times)\n",
    "    \n",
    "    results = pd.DataFrame(columns = [f'inf1_compiled_batch_size_{batch_size}_compiled_cores_{num_cores}'])\n",
    "    results.loc['instance_type']           = [requests.get('http://169.254.169.254/latest/meta-data/instance-type').text]\n",
    "    results.loc['compiled_batch_size']     = [batch_size]\n",
    "    results.loc['user_batch_size']         = [user_batch_size]\n",
    "    results.loc['accuracy']                = [acc_inf1]\n",
    "    results.loc['prediction_time']         = [np.sum(iter_times)]\n",
    "    results.loc['wall_time']               = [time.time() - walltime_start]\n",
    "    results.loc['images_per_sec_mean']     = [np.mean(user_batch_size / iter_times)]\n",
    "    results.loc['images_per_sec_std']      = [np.std(user_batch_size / iter_times, ddof=1)]\n",
    "    results.loc['latency_mean']            = [np.mean(iter_times) * 1000]\n",
    "    results.loc['latency_99th_percentile'] = [np.percentile(iter_times, q=99, interpolation=\"lower\") * 1000]\n",
    "    results.loc['latency_median']          = [np.median(iter_times) * 1000]\n",
    "    results.loc['latency_min']             = [np.min(iter_times) * 1000]\n",
    "    display(results.T)\n",
    "\n",
    "    return results, iter_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------batch size: 1, num cores: 1----------\n",
      "Compiling...\n",
      "INFO:tensorflow:Restoring parameters from resnet50_saved_model/variables/variables\n",
      "INFO:tensorflow:Froze 320 variables.\n",
      "INFO:tensorflow:Converted 320 variables to const ops.\n",
      "INFO:tensorflow:fusing subgraph neuron_op_d6f098c01c780733 with neuron-cc; log file is at /home/ubuntu/examples/bkp/ai-accelerators-examples/compiler-workdir/resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_1/neuron_op_d6f098c01c780733/graph_def.neuron-cc.log\n",
      "INFO:tensorflow:Number of operations in TensorFlow session: 4647\n",
      "INFO:tensorflow:Number of operations after tf.neuron optimizations: 556\n",
      "INFO:tensorflow:Number of operations placed on Neuron runtime: 554\n",
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_1/saved_model.pb\n",
      "INFO:tensorflow:Successfully converted resnet50_saved_model to resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_1\n",
      "Compile time: 57.83445167541504\n",
      "resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_1\n",
      "{'OnNeuronRatio': 0.9964028776978417}\n",
      "----------- Done! ----------- \n",
      "\n",
      "-----------batch size: 5, num cores: 1----------\n",
      "Compiling...\n",
      "INFO:tensorflow:Restoring parameters from resnet50_saved_model/variables/variables\n",
      "INFO:tensorflow:Froze 320 variables.\n",
      "INFO:tensorflow:Converted 320 variables to const ops.\n",
      "INFO:tensorflow:fusing subgraph neuron_op_d6f098c01c780733 with neuron-cc; log file is at /home/ubuntu/examples/bkp/ai-accelerators-examples/compiler-workdir/resnet50_inf1_saved_models/resnet50_batch_5_inf1_cores_1/neuron_op_d6f098c01c780733/graph_def.neuron-cc.log\n",
      "INFO:tensorflow:Number of operations in TensorFlow session: 4647\n",
      "INFO:tensorflow:Number of operations after tf.neuron optimizations: 556\n",
      "INFO:tensorflow:Number of operations placed on Neuron runtime: 554\n",
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: resnet50_inf1_saved_models/resnet50_batch_5_inf1_cores_1/saved_model.pb\n",
      "INFO:tensorflow:Successfully converted resnet50_saved_model to resnet50_inf1_saved_models/resnet50_batch_5_inf1_cores_1\n",
      "Compile time: 96.24623918533325\n",
      "resnet50_inf1_saved_models/resnet50_batch_5_inf1_cores_1\n",
      "{'OnNeuronRatio': 0.9964028776978417}\n",
      "----------- Done! ----------- \n",
      "\n",
      "-----------batch size: 1, num cores: 4----------\n",
      "Compiling...\n",
      "INFO:tensorflow:Restoring parameters from resnet50_saved_model/variables/variables\n",
      "INFO:tensorflow:Froze 320 variables.\n",
      "INFO:tensorflow:Converted 320 variables to const ops.\n",
      "INFO:tensorflow:fusing subgraph neuron_op_d6f098c01c780733 with neuron-cc; log file is at /home/ubuntu/examples/bkp/ai-accelerators-examples/compiler-workdir/resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_4/neuron_op_d6f098c01c780733/graph_def.neuron-cc.log\n",
      "INFO:tensorflow:Number of operations in TensorFlow session: 4647\n",
      "INFO:tensorflow:Number of operations after tf.neuron optimizations: 556\n",
      "INFO:tensorflow:Number of operations placed on Neuron runtime: 554\n",
      "INFO:tensorflow:No assets to save.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_4/saved_model.pb\n",
      "INFO:tensorflow:Successfully converted resnet50_saved_model to resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_4\n",
      "Compile time: 63.46157956123352\n",
      "resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_4\n",
      "{'OnNeuronRatio': 0.9964028776978417}\n",
      "----------- Done! ----------- \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inf1_model_dir = 'resnet50_inf1_saved_models'\n",
    "saved_model_dir = 'resnet50_saved_model'\n",
    "\n",
    "compile_inf1_model(saved_model_dir, inf1_model_dir, batch_size=1, num_cores=1)\n",
    "compile_inf1_model(saved_model_dir, inf1_model_dir, batch_size=5, num_cores=1)\n",
    "compile_inf1_model(saved_model_dir, inf1_model_dir, batch_size=1, num_cores=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inf1_compiled_model_dir: resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_1\n",
      "Running model resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_1, user_batch_size: 10\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/aws_neuron_tensorflow_p36/lib/python3.6/site-packages/tensorflow_core/contrib/predictor/saved_model_predictor.py:153: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:The specified SavedModel has no variables; no checkpoints were restored.\n",
      "WARNING:tensorflow:From <ipython-input-7-6f77d9401466>:14: DatasetV1.make_initializable_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_initializable_iterator(dataset)`.\n",
      "Images 5000/50000. Average i/s 511.9149005447497\n",
      "Images 10000/50000. Average i/s 514.1476835875276\n",
      "Images 15000/50000. Average i/s 511.55752611295105\n",
      "Images 20000/50000. Average i/s 510.6258382445502\n",
      "Images 25000/50000. Average i/s 510.6002877210464\n",
      "Images 30000/50000. Average i/s 510.33624960724376\n",
      "Images 35000/50000. Average i/s 510.23169540573906\n",
      "Images 40000/50000. Average i/s 509.62934416741103\n",
      "Images 45000/50000. Average i/s 509.851016680137\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instance_type</th>\n",
       "      <th>compiled_batch_size</th>\n",
       "      <th>user_batch_size</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>prediction_time</th>\n",
       "      <th>wall_time</th>\n",
       "      <th>images_per_sec_mean</th>\n",
       "      <th>images_per_sec_std</th>\n",
       "      <th>latency_mean</th>\n",
       "      <th>latency_99th_percentile</th>\n",
       "      <th>latency_median</th>\n",
       "      <th>latency_min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>inf1_compiled_batch_size_1_compiled_cores_1</th>\n",
       "      <td>inf1.xlarge</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.74852</td>\n",
       "      <td>99.4977</td>\n",
       "      <td>146.44</td>\n",
       "      <td>509.898</td>\n",
       "      <td>59.3509</td>\n",
       "      <td>19.8995</td>\n",
       "      <td>27.0095</td>\n",
       "      <td>19.4516</td>\n",
       "      <td>15.7409</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            instance_type compiled_batch_size  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_1   inf1.xlarge                   1   \n",
       "\n",
       "                                            user_batch_size accuracy  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_1              10  0.74852   \n",
       "\n",
       "                                            prediction_time wall_time  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_1         99.4977    146.44   \n",
       "\n",
       "                                            images_per_sec_mean  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_1             509.898   \n",
       "\n",
       "                                            images_per_sec_std latency_mean  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_1            59.3509      19.8995   \n",
       "\n",
       "                                            latency_99th_percentile  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_1                 27.0095   \n",
       "\n",
       "                                            latency_median latency_min  \n",
       "inf1_compiled_batch_size_1_compiled_cores_1        19.4516     15.7409  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inf1_compiled_model_dir: resnet50_inf1_saved_models/resnet50_batch_5_inf1_cores_1\n",
      "Running model resnet50_inf1_saved_models/resnet50_batch_5_inf1_cores_1, user_batch_size: 50\n",
      "\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:The specified SavedModel has no variables; no checkpoints were restored.\n",
      "Images 5000/50000. Average i/s 756.9368826530499\n",
      "Images 10000/50000. Average i/s 758.3658209888853\n",
      "Images 15000/50000. Average i/s 758.889930748228\n",
      "Images 20000/50000. Average i/s 760.4266183337617\n",
      "Images 25000/50000. Average i/s 760.8172446072302\n",
      "Images 30000/50000. Average i/s 760.2700641837997\n",
      "Images 35000/50000. Average i/s 759.5763119031232\n",
      "Images 40000/50000. Average i/s 760.0876622928208\n",
      "Images 45000/50000. Average i/s 760.6692409949743\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instance_type</th>\n",
       "      <th>compiled_batch_size</th>\n",
       "      <th>user_batch_size</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>prediction_time</th>\n",
       "      <th>wall_time</th>\n",
       "      <th>images_per_sec_mean</th>\n",
       "      <th>images_per_sec_std</th>\n",
       "      <th>latency_mean</th>\n",
       "      <th>latency_99th_percentile</th>\n",
       "      <th>latency_median</th>\n",
       "      <th>latency_min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>inf1_compiled_batch_size_5_compiled_cores_1</th>\n",
       "      <td>inf1.xlarge</td>\n",
       "      <td>5</td>\n",
       "      <td>50</td>\n",
       "      <td>0.7486</td>\n",
       "      <td>66.1318</td>\n",
       "      <td>143.338</td>\n",
       "      <td>760.852</td>\n",
       "      <td>59.7863</td>\n",
       "      <td>66.1318</td>\n",
       "      <td>80.0529</td>\n",
       "      <td>65.8345</td>\n",
       "      <td>56.5889</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            instance_type compiled_batch_size  \\\n",
       "inf1_compiled_batch_size_5_compiled_cores_1   inf1.xlarge                   5   \n",
       "\n",
       "                                            user_batch_size accuracy  \\\n",
       "inf1_compiled_batch_size_5_compiled_cores_1              50   0.7486   \n",
       "\n",
       "                                            prediction_time wall_time  \\\n",
       "inf1_compiled_batch_size_5_compiled_cores_1         66.1318   143.338   \n",
       "\n",
       "                                            images_per_sec_mean  \\\n",
       "inf1_compiled_batch_size_5_compiled_cores_1             760.852   \n",
       "\n",
       "                                            images_per_sec_std latency_mean  \\\n",
       "inf1_compiled_batch_size_5_compiled_cores_1            59.7863      66.1318   \n",
       "\n",
       "                                            latency_99th_percentile  \\\n",
       "inf1_compiled_batch_size_5_compiled_cores_1                 80.0529   \n",
       "\n",
       "                                            latency_median latency_min  \n",
       "inf1_compiled_batch_size_5_compiled_cores_1        65.8345     56.5889  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inf1_compiled_model_dir: resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_4\n",
      "Running model resnet50_inf1_saved_models/resnet50_batch_1_inf1_cores_4, user_batch_size: 10\n",
      "\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:The specified SavedModel has no variables; no checkpoints were restored.\n",
      "Images 5000/50000. Average i/s 361.0220527406529\n",
      "Images 10000/50000. Average i/s 359.68171067062457\n",
      "Images 15000/50000. Average i/s 358.75999045862824\n",
      "Images 20000/50000. Average i/s 358.28821764290626\n",
      "Images 25000/50000. Average i/s 357.87710423546616\n",
      "Images 30000/50000. Average i/s 358.12412006408044\n",
      "Images 35000/50000. Average i/s 358.09136022506436\n",
      "Images 40000/50000. Average i/s 357.7498270103637\n",
      "Images 45000/50000. Average i/s 357.8152796870437\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instance_type</th>\n",
       "      <th>compiled_batch_size</th>\n",
       "      <th>user_batch_size</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>prediction_time</th>\n",
       "      <th>wall_time</th>\n",
       "      <th>images_per_sec_mean</th>\n",
       "      <th>images_per_sec_std</th>\n",
       "      <th>latency_mean</th>\n",
       "      <th>latency_99th_percentile</th>\n",
       "      <th>latency_median</th>\n",
       "      <th>latency_min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>inf1_compiled_batch_size_1_compiled_cores_4</th>\n",
       "      <td>inf1.xlarge</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.749</td>\n",
       "      <td>140.423</td>\n",
       "      <td>147.618</td>\n",
       "      <td>357.974</td>\n",
       "      <td>25.1203</td>\n",
       "      <td>28.0846</td>\n",
       "      <td>34.502</td>\n",
       "      <td>27.5503</td>\n",
       "      <td>25.4283</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            instance_type compiled_batch_size  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_4   inf1.xlarge                   1   \n",
       "\n",
       "                                            user_batch_size accuracy  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_4              10    0.749   \n",
       "\n",
       "                                            prediction_time wall_time  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_4         140.423   147.618   \n",
       "\n",
       "                                            images_per_sec_mean  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_4             357.974   \n",
       "\n",
       "                                            images_per_sec_std latency_mean  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_4            25.1203      28.0846   \n",
       "\n",
       "                                            latency_99th_percentile  \\\n",
       "inf1_compiled_batch_size_1_compiled_cores_4                  34.502   \n",
       "\n",
       "                                            latency_median latency_min  \n",
       "inf1_compiled_batch_size_1_compiled_cores_4        27.5503     25.4283  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inf1_compiled_batch_size_1_compiled_cores_1</th>\n",
       "      <th>inf1_compiled_batch_size_5_compiled_cores_1</th>\n",
       "      <th>inf1_compiled_batch_size_1_compiled_cores_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>instance_type</th>\n",
       "      <td>inf1.xlarge</td>\n",
       "      <td>inf1.xlarge</td>\n",
       "      <td>inf1.xlarge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>compiled_batch_size</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_batch_size</th>\n",
       "      <td>10</td>\n",
       "      <td>50</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.74852</td>\n",
       "      <td>0.7486</td>\n",
       "      <td>0.749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prediction_time</th>\n",
       "      <td>99.4977</td>\n",
       "      <td>66.1318</td>\n",
       "      <td>140.423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wall_time</th>\n",
       "      <td>146.44</td>\n",
       "      <td>143.338</td>\n",
       "      <td>147.618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>images_per_sec_mean</th>\n",
       "      <td>509.898</td>\n",
       "      <td>760.852</td>\n",
       "      <td>357.974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>images_per_sec_std</th>\n",
       "      <td>59.3509</td>\n",
       "      <td>59.7863</td>\n",
       "      <td>25.1203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_mean</th>\n",
       "      <td>19.8995</td>\n",
       "      <td>66.1318</td>\n",
       "      <td>28.0846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_99th_percentile</th>\n",
       "      <td>27.0095</td>\n",
       "      <td>80.0529</td>\n",
       "      <td>34.502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_median</th>\n",
       "      <td>19.4516</td>\n",
       "      <td>65.8345</td>\n",
       "      <td>27.5503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_min</th>\n",
       "      <td>15.7409</td>\n",
       "      <td>56.5889</td>\n",
       "      <td>25.4283</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        inf1_compiled_batch_size_1_compiled_cores_1  \\\n",
       "instance_type                                           inf1.xlarge   \n",
       "compiled_batch_size                                               1   \n",
       "user_batch_size                                                  10   \n",
       "accuracy                                                    0.74852   \n",
       "prediction_time                                             99.4977   \n",
       "wall_time                                                    146.44   \n",
       "images_per_sec_mean                                         509.898   \n",
       "images_per_sec_std                                          59.3509   \n",
       "latency_mean                                                19.8995   \n",
       "latency_99th_percentile                                     27.0095   \n",
       "latency_median                                              19.4516   \n",
       "latency_min                                                 15.7409   \n",
       "\n",
       "                        inf1_compiled_batch_size_5_compiled_cores_1  \\\n",
       "instance_type                                           inf1.xlarge   \n",
       "compiled_batch_size                                               5   \n",
       "user_batch_size                                                  50   \n",
       "accuracy                                                     0.7486   \n",
       "prediction_time                                             66.1318   \n",
       "wall_time                                                   143.338   \n",
       "images_per_sec_mean                                         760.852   \n",
       "images_per_sec_std                                          59.7863   \n",
       "latency_mean                                                66.1318   \n",
       "latency_99th_percentile                                     80.0529   \n",
       "latency_median                                              65.8345   \n",
       "latency_min                                                 56.5889   \n",
       "\n",
       "                        inf1_compiled_batch_size_1_compiled_cores_4  \n",
       "instance_type                                           inf1.xlarge  \n",
       "compiled_batch_size                                               1  \n",
       "user_batch_size                                                  10  \n",
       "accuracy                                                      0.749  \n",
       "prediction_time                                             140.423  \n",
       "wall_time                                                   147.618  \n",
       "images_per_sec_mean                                         357.974  \n",
       "images_per_sec_std                                          25.1203  \n",
       "latency_mean                                                28.0846  \n",
       "latency_99th_percentile                                      34.502  \n",
       "latency_median                                              27.5503  \n",
       "latency_min                                                 25.4283  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "inf1_model_dir = 'resnet50_inf1_saved_models'\n",
    "\n",
    "compile_options = [{'batch_size': 1, 'num_cores': 1},\n",
    "                  {'batch_size': 5, 'num_cores': 1},\n",
    "                  {'batch_size': 1, 'num_cores': 4}]\n",
    "\n",
    "iter_ds = pd.DataFrame()\n",
    "results = pd.DataFrame()\n",
    "\n",
    "for opt in compile_options:\n",
    "    batch_size = opt[\"batch_size\"]\n",
    "    num_cores = opt[\"num_cores\"]\n",
    "    compiled_model_dir = f'resnet50_batch_{batch_size}_inf1_cores_{num_cores}'\n",
    "    inf1_compiled_model_dir = os.path.join(inf1_model_dir, compiled_model_dir)\n",
    "   \n",
    "    print(f'inf1_compiled_model_dir: {inf1_compiled_model_dir}')\n",
    "    col_name = lambda opt: f'inf1_{batch_size}_multicores_{num_cores}'\n",
    "    \n",
    "    res, iter_times = inf1_predict_benchmark_single_threaded(inf1_compiled_model_dir,\n",
    "                                                                     batch_size = batch_size,\n",
    "                                                                     user_batch_size = batch_size*10,\n",
    "                                                                     num_cores = num_cores,\n",
    "                                                                     use_cache=False, \n",
    "                                                                     warm_up=10)\n",
    "    \n",
    "    iter_ds = pd.concat([iter_ds, pd.DataFrame(iter_times, columns=[col_name(opt)])], axis=1)\n",
    "    results = pd.concat([results, res], axis=1)\n",
    "    \n",
    "display(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_aws_neuron_tensorflow_p36)",
   "language": "python",
   "name": "conda_aws_neuron_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}