evaluations/model/SFNDelay/StepFunctionsDelay.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# StepFunctionDelay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*  A small scale App to measure delay in Step Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from io import BytesIO\n",
    "import time\n",
    "import zipfile\n",
    "import numpy as np\n",
    "import boto3\n",
    "from tqdm import tqdm\n",
    "from datetime import datetime, timezone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = boto3.client('lambda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "function_prefix='StepFunctionDelay'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* All functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['serverlessrepo-DocumentGenerator-MyFunction-9MVH750993AD',\n",
       " 'ServerlessAppPerfOpt_f5',\n",
       " 'ServerlessAppPerfOpt_f4',\n",
       " 'ServerlessAppPerfOpt_f6',\n",
       " 'ServerlessAppPerfOpt_f2',\n",
       " 'ServerlessAppPerfOpt_f3',\n",
       " 'ServerlessAppPerfOpt_f1']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "functions=client.list_functions()\n",
    "function_name=[item['FunctionName'] for item in functions['Functions']]\n",
    "function_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update all Functions in StepFunctionDelay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update Function Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "functions=[]\n",
    "for file in os.listdir('functions'):\n",
    "    path=os.path.abspath(os.path.join(os.path.dirname('__file__'), 'functions/'+file))\n",
    "    if not file.startswith('.') and os.path.isdir(path):\n",
    "        functions.append(file)\n",
    "for function_folder in functions:\n",
    "    buf = BytesIO()\n",
    "    with zipfile.ZipFile(buf, 'w') as z:\n",
    "        for file in os.listdir('functions/'+function_folder):\n",
    "            z.write(os.path.abspath(os.path.join(os.path.dirname('__file__'), 'functions/{}/{}'.format(function_folder,file))), os.path.basename(os.path.join(os.path.dirname('__file__'), 'functions/{}/{}'.format(function_folder,file))))\n",
    "    buf.seek(0)\n",
    "    pkg = buf.read()\n",
    "    client.update_function_code(FunctionName='{}_{}'.format(function_prefix, function_folder),ZipFile=pkg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update Function Memory Configuration\n",
    "* Available Memory Configurations: 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664, 1728, 1792, 1856, 1920, 1984, 2048, 2112, 2176, 2240, 2304, 2368, 2432, 2496, 2560, 2624, 2688, 2752, 2816, 2880, 2944, 3008\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "mem_config_list={\n",
    "    'f1':128,\n",
    "    'f2':128,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for function in mem_config_list.keys():\n",
    "    client.update_function_configuration(FunctionName='{}_{}'.format(function_prefix, function), MemorySize=mem_config_list[function])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfn_client = boto3.client('stepfunctions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_machines=sfn_client.list_state_machines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'creationDate': datetime.datetime(2019, 12, 29, 22, 21, 52, 242000, tzinfo=tzlocal()),\n",
       "  'name': 'HelloWorld',\n",
       "  'stateMachineArn': 'arn:aws:states:us-east-2:499537426559:stateMachine:HelloWorld',\n",
       "  'type': 'STANDARD'},\n",
       " {'creationDate': datetime.datetime(2019, 12, 29, 22, 32, 7, 221000, tzinfo=tzlocal()),\n",
       "  'name': 'ServerlessAppPerfOpt_AppStepFunction',\n",
       "  'stateMachineArn': 'arn:aws:states:us-east-2:499537426559:stateMachine:ServerlessAppPerfOpt_AppStepFunction',\n",
       "  'type': 'EXPRESS'},\n",
       " {'creationDate': datetime.datetime(2019, 12, 30, 21, 56, 22, 479000, tzinfo=tzlocal()),\n",
       "  'name': 'StepFunctionDelay',\n",
       "  'stateMachineArn': 'arn:aws:states:us-east-2:499537426559:stateMachine:StepFunctionDelay',\n",
       "  'type': 'EXPRESS'}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_machines['stateMachines']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "stateMachineArn='arn:aws:states:us-east-2:499537426559:stateMachine:StepFunctionDelay'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Execute StepFunctionDelay 100 times with 10-second interval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Execute StepFunctionDelay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [16:54<00:00, 10.15s/it]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(100)):\n",
    "    time.sleep(10)\n",
    "    sfn_client.start_execution(stateMachineArn=stateMachineArn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Execute App1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [16:53<00:00, 10.16s/it]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(100)):\n",
    "    time.sleep(10)\n",
    "    sfn_client.start_execution(stateMachineArn='arn:aws:states:us-east-2:499537426559:stateMachine:ServerlessAppPerfOpt_AppStepFunction')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": false
   },
   "source": [
    "## CloudWatch Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "logclient = boto3.client('logs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lambda_report_log_to_dict(log):\n",
    "    res={}\n",
    "    lis=[item.split(': ') for item in log.split('\\t')]\n",
    "    res['RequestId']=lis[0][1]\n",
    "    res['Duration']=float(lis[1][1].split(' ')[0])\n",
    "    res['Billed Duration']=int(lis[2][1].split(' ')[0])\n",
    "    res['Memory Size']=int(lis[3][1].split(' ')[0])\n",
    "    res['Max Memory Used']=int(lis[4][1].split(' ')[0])\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Query Step Function Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "startTime_sfndelay=int(datetime.timestamp(datetime(year=2019,month=12,day=31,hour=5,minute=8,second=0,tzinfo=timezone.utc)))\n",
    "endTime_sfndelay=int(datetime.timestamp(datetime(year=2019,month=12,day=31,hour=5,minute=26,second=0,tzinfo=timezone.utc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_sfndelay = logclient.start_query(\n",
    "    logGroupName='/aws/states/{}-Logs'.format(function_prefix),\n",
    "    queryString=\"fields id, @timestamp| filter id = 4 or id = 5 or id = 9 or id = 10| sort id desc\",\n",
    "    startTime=startTime_sfndelay,\n",
    "    endTime=endTime_sfndelay\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_results_sfndelay = logclient.get_query_results(\n",
    "    queryId=query_sfndelay['queryId']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfndelay['results'] if item[0]['value']=='4'])\n",
    "f1_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfndelay['results'] if item[0]['value']=='5'])\n",
    "f2_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfndelay['results'] if item[0]['value']=='9'])\n",
    "f2_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfndelay['results'] if item[0]['value']=='10'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfn_function_duration_f1=(f1_endtimestamp-f1_starttimestamp)*1000\n",
    "sfn_function_duration_f2=(f2_endtimestamp-f2_starttimestamp)*1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Query Lambda Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_f1 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format(function_prefix,'f1'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_sfndelay,\n",
    "    endTime=endTime_sfndelay\n",
    ")\n",
    "query_lambdaduration_f2 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format(function_prefix,'f2'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_sfndelay,\n",
    "    endTime=endTime_sfndelay\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_results_lambdaduration_f1 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_f1['queryId']\n",
    ")\n",
    "query_results_lambdaduration_f2 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_f2['queryId']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_f1['results']]]\n",
    "f2_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_f2['results']]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Delay Incurred by Step Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  404.033 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  4.402512805205684 ms\n",
      "Average Duration Reported by Step Function:  428.12999963760376 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  11.390062562424406 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  24.09699963760376 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  10.725752062646826 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(f1_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(f1_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(f1_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_f1), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_f1), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_f1-np.array(f1_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_f1-np.array(f1_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  203.3043 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  3.640076992317607 ms\n",
      "Average Duration Reported by Step Function:  220.92000722885132 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  6.3713097529368286 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  17.61570722885132 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  5.367679049879902 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(f2_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(f2_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(f2_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_f2), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_f2), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_f2-np.array(f2_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_f2-np.array(f2_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Delay Incurred by Step Function - App1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Query Step Function Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "startTime_App1=int(datetime.timestamp(datetime(year=2019,month=12,day=31,hour=5,minute=32,second=0,tzinfo=timezone.utc)))\n",
    "endTime_App1=int(datetime.timestamp(datetime(year=2019,month=12,day=31,hour=5,minute=49,second=0,tzinfo=timezone.utc)))\n",
    "query_sfnApp1delay = logclient.start_query(\n",
    "    logGroupName='/aws/states/{}-Logs'.format('ServerlessAppPerfOpt_AppStepFunction'),\n",
    "    queryString=\"fields id, @timestamp| filter id = 4 or id = 5 or id = 9 or id = 10 or id = 16 or id = 19 or id = 20 or id = 22 or id = 28 or id = 29 or id = 33 or id = 34| sort id desc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1,\n",
    "    limit=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_results_sfnApp1delay = logclient.get_query_results(\n",
    "    queryId=query_sfnApp1delay['queryId']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "App1_f1_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='4'])\n",
    "App1_f1_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='5'])\n",
    "App1_f2_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='9'])\n",
    "App1_f2_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='10'])\n",
    "App1_f3_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='16'])\n",
    "App1_f3_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='20'])\n",
    "App1_f4_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='19'])\n",
    "App1_f4_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='22'])\n",
    "App1_f5_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='28'])\n",
    "App1_f5_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='29'])\n",
    "App1_f6_starttimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='33'])\n",
    "App1_f6_endtimestamp=np.sort([datetime.timestamp(datetime.strptime(item[1]['value'], '%Y-%m-%d %H:%M:%S.%f')) for item in query_results_sfnApp1delay['results'] if item[0]['value']=='34'])\n",
    "sfn_function_duration_App1_f1=(App1_f1_endtimestamp-App1_f1_starttimestamp)*1000\n",
    "sfn_function_duration_App1_f2=(App1_f2_endtimestamp-App1_f2_starttimestamp)*1000\n",
    "sfn_function_duration_App1_f3=(App1_f3_endtimestamp-App1_f3_starttimestamp)*1000\n",
    "sfn_function_duration_App1_f4=(App1_f4_endtimestamp-App1_f4_starttimestamp)*1000\n",
    "sfn_function_duration_App1_f5=(App1_f5_endtimestamp-App1_f5_starttimestamp)*1000\n",
    "sfn_function_duration_App1_f6=(App1_f6_endtimestamp-App1_f6_starttimestamp)*1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Query Lambda Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_App1_f1 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format('ServerlessAppPerfOpt','f1'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_App1_f2 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format('ServerlessAppPerfOpt','f2'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_App1_f3 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format('ServerlessAppPerfOpt','f3'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_App1_f4 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format('ServerlessAppPerfOpt','f4'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_App1_f5 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format('ServerlessAppPerfOpt','f5'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_lambdaduration_App1_f6 = logclient.start_query(\n",
    "    logGroupName='/aws/lambda/{}_{}'.format('ServerlessAppPerfOpt','f6'),\n",
    "    queryString=\"fields @timestamp, @message| filter @message like 'REPORT'| sort @timestamp asc\",\n",
    "    startTime=startTime_App1,\n",
    "    endTime=endTime_App1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_results_lambdaduration_App1_f1 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_App1_f1['queryId']\n",
    ")\n",
    "App1_f1_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_App1_f1['results']]]\n",
    "query_results_lambdaduration_App1_f2 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_App1_f2['queryId']\n",
    ")\n",
    "App1_f2_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_App1_f2['results']]]\n",
    "query_results_lambdaduration_App1_f3 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_App1_f3['queryId']\n",
    ")\n",
    "App1_f3_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_App1_f3['results']]]\n",
    "query_results_lambdaduration_App1_f4 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_App1_f4['queryId']\n",
    ")\n",
    "App1_f4_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_App1_f4['results']]]\n",
    "query_results_lambdaduration_App1_f5 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_App1_f5['queryId']\n",
    ")\n",
    "App1_f5_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_App1_f5['results']]]\n",
    "query_results_lambdaduration_App1_f6 = logclient.get_query_results(\n",
    "    queryId=query_lambdaduration_App1_f6['queryId']\n",
    ")\n",
    "App1_f6_lamdaduration_list=[item['Duration'] for item in [lambda_report_log_to_dict(item [1]['value']) for item in query_results_lambdaduration_App1_f6['results']]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  403.7752 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  4.185725380385101 ms\n",
      "Average Duration Reported by Step Function:  430.18998622894287 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  35.98907378409499 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  26.41478622894288 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  35.996724710417524 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(App1_f1_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(App1_f1_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(App1_f1_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_App1_f1), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_App1_f1), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_App1_f1-np.array(App1_f1_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_App1_f1-np.array(App1_f1_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  203.03099999999998 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  3.0641339722668786 ms\n",
      "Average Duration Reported by Step Function:  220.72999715805054 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  20.476272049985642 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  17.69899715805054 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  20.330270199515617 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(App1_f2_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(App1_f2_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(App1_f2_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_App1_f2), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_App1_f2), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_App1_f2-np.array(App1_f2_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_App1_f2-np.array(App1_f2_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  804.2271000000001 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  4.207432541348701 ms\n",
      "Average Duration Reported by Step Function:  820.3500175476074 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  19.749113067043325 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  16.122917547607422 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  19.316455176125064 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(App1_f3_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(App1_f3_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(App1_f3_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_App1_f3), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_App1_f3), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_App1_f3-np.array(App1_f3_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_App1_f3-np.array(App1_f3_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  1003.6695000000001 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  3.2994482493289725 ms\n",
      "Average Duration Reported by Step Function:  1022.1100091934204 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  21.139969033778595 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  18.440509193420407 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  20.890728082924664 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(App1_f4_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(App1_f4_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(App1_f4_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_App1_f4), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_App1_f4), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_App1_f4-np.array(App1_f4_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_App1_f4-np.array(App1_f4_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  503.8387 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  3.996418810635337 ms\n",
      "Average Duration Reported by Step Function:  520.8300065994263 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  19.920870145359665 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  16.991306599426267 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  19.67841544632416 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(App1_f5_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(App1_f5_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(App1_f5_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_App1_f5), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_App1_f5), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_App1_f5-np.array(App1_f5_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_App1_f5-np.array(App1_f5_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Executions:  100\n",
      "Average Duration Reported by Lambda:  603.2264 ms\n",
      "Standard Deviation of Duration Reported by Lambda:  2.6983763710794637 ms\n",
      "Average Duration Reported by Step Function:  620.4499983787537 ms\n",
      "Standard Deviation of Duration Reported by Step Function:  24.042197570967733 ms\n",
      "Average Discrepancy between Durations reported by Step Function and Lambda:  17.22359837875366 ms\n",
      "Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda:  23.9623726378212 ms\n"
     ]
    }
   ],
   "source": [
    "print('Number of Executions: ', len(App1_f6_lamdaduration_list))\n",
    "print('Average Duration Reported by Lambda: ', np.mean(App1_f6_lamdaduration_list), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Lambda: ', np.std(App1_f6_lamdaduration_list), 'ms')\n",
    "print('Average Duration Reported by Step Function: ', np.mean(sfn_function_duration_App1_f6), 'ms')\n",
    "print('Standard Deviation of Duration Reported by Step Function: ', np.std(sfn_function_duration_App1_f6), 'ms')\n",
    "print('Average Discrepancy between Durations reported by Step Function and Lambda: ', np.mean(sfn_function_duration_App1_f6-np.array(App1_f6_lamdaduration_list)), 'ms')\n",
    "print('Standard Deviation of Discrepancy between Durations reported by Step Function and Lambda: ', np.std(sfn_function_duration_App1_f6-np.array(App1_f6_lamdaduration_list)), 'ms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "discrepancy_App1_f1 = sfn_function_duration_App1_f1-np.array(App1_f1_lamdaduration_list)\n",
    "discrepancy_App1_f2 = sfn_function_duration_App1_f2-np.array(App1_f2_lamdaduration_list)\n",
    "discrepancy_App1_f3 = sfn_function_duration_App1_f3-np.array(App1_f3_lamdaduration_list)\n",
    "discrepancy_App1_f4 = sfn_function_duration_App1_f4-np.array(App1_f4_lamdaduration_list)\n",
    "discrepancy_App1_f5 = sfn_function_duration_App1_f5-np.array(App1_f5_lamdaduration_list)\n",
    "discrepancy_App1_f6 = sfn_function_duration_App1_f6-np.array(App1_f6_lamdaduration_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "discrepancy_App1 = np.concatenate((discrepancy_App1_f1, discrepancy_App1_f2, discrepancy_App1_f3, discrepancy_App1_f4, discrepancy_App1_f5, discrepancy_App1_f6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18.815352517700198"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(discrepancy_App1) # AWS Step Functions Interaction Delay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5.2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}