Skip to content

Commit

Permalink
added preprocess materials
Browse files Browse the repository at this point in the history
  • Loading branch information
@sajibtariq
sajibtariq committed May 3, 2023
1 parent 3b7ded1 commit d0fba36
Show file tree
Hide file tree
Showing 28 changed files with 17,081 additions and 0 deletions.
4,904 changes: 4,904 additions & 0 deletions Data_Preprocess/Final_Dataset_Sample/HTTPS.csv
View file

Large diffs are not rendered by default.

1,813 changes: 1,813 additions & 0 deletions Data_Preprocess/Final_Dataset_Sample/QUIC.csv
View file

Large diffs are not rendered by default.

267 changes: 267 additions & 0 deletions Data_Preprocess/QoE_QoS_Merge/QoE-QoS_merge-HTTPS(TCP).ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,267 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "aa8d4787",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import warnings\n",
"warnings.filterwarnings(\"ignore\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3940737f",
"metadata": {},
"outputs": [],
"source": [
"df= pd.DataFrame(columns= [\n",
" 'Iteration', 'Type', 'Mobility', 'Delay', 'Tp_type', 'Ap_type', 'Server_type', 'V_id', 'Tile_scheme', \n",
" 'VP_trace', 'VP_error', 'Thread', 'Total_seg', 'Algo', 'Buffer_time', 'Host_num', 'Bg_traffic',\n",
"\n",
" 'til_720_z1', ' til_1080_z1', ' til_4k_z1', ' til_720_z2',\n",
" ' til_1080_z2', ' til_4k_z2', ' til_720_z3', ' til_1080_z3',\n",
" 'til_4k_z3', ' z1_bit', ' z2_bit', ' z3_bit', ' qt_sw_z1', ' qt_sw_z2',\n",
" ' qt_sw_z3', ' total_stall', ' start_time', 'QoE', 'QoE1', \n",
"\n",
"\n",
" 'TP_D_avg', 'TP_D_max', 'TP_D_min', 'TP_D_medn', 'TP_D_std', \n",
" 'TP_D_10p', 'TP_D_20p', 'TP_D_30p', 'TP_D_40p', 'TP_D_50p', 'TP_D_60p', 'TP_D_70p', 'TP_D_80p', 'TP_D_90p',\n",
" 'TP_D_F25_avg', 'TP_D_F50_avg', 'TP_D_L25_avg', 'TP_D_L50_avg',\n",
" \n",
" \n",
" 'TP_D100_avg', 'TP_D100_max', 'TP_D100_min', 'TP_D100_medn', 'TP_D100_std', \n",
" 'TP_D100_10p', 'TP_D100_20p', 'TP_D100_30p', 'TP_D100_40p', 'TP_D100_50p', 'TP_D100_60p', 'TP_D100_70p', 'TP_D100_80p', 'TP_D100_90p',\n",
" 'TP_D100_F25_avg', 'TP_D100_F50_avg', 'TP_D100_L25_avg', 'TP_D100_L50_avg',\n",
" \n",
" \n",
" 'TP_U_avg', 'TP_U_max', 'TP_U_min', 'TP_U_medn', 'TP_U_std', \n",
" 'TP_U_10p', 'TP_U_20p', 'TP_U_30p', 'TP_U_40p', 'TP_U_50p', 'TP_U_60p', 'TP_U_70p', 'TP_U_80p', 'TP_U_90p',\n",
" 'TP_U_F25_avg', 'TP_U_F50_avg', 'TP_U_L25_avg', 'TP_U_L50_avg',\n",
" \n",
" 'TP_U100_avg', 'TP_U100_max', 'TP_U100_min', 'TP_U100_medn', 'TP_U100_std', \n",
" 'TP_U100_10p', 'TP_U100_20p', 'TP_U100_30p', 'TP_U100_40p', 'TP_U100_50p', 'TP_U100_60p', 'TP_U100_70p', 'TP_U100_80p', 'TP_U100_90p',\n",
" 'TP_U100_F25_avg', 'TP_U100_F50_avg', 'TP_U100_L25_avg', 'TP_U100_L50_avg',\n",
"\n",
" 'PC_D_total', 'PC_D_avg', 'PC_D_max', 'PC_D_min', 'PC_D_medn', 'PC_D_std', \n",
" 'PC_D_10p', 'PC_D_20p', 'PC_D_30p', 'PC_D_40p', 'PC_D_50p', 'PC_D_60p', 'PC_D_70p', 'PC_D_80p', 'PC_D_90p',\n",
" 'PC_D_F25_total', 'TP_D_F50_total', 'PC_D_L25_total', 'PC_D_L50_total',\n",
" \n",
" 'PC_D100_total', 'PC_D100_avg', 'PC_D100_max', 'PC_D100_min', 'PC_D100_medn', 'PC_D100_std', \n",
" 'PC_D100_10p', 'PC_D100_20p', 'PC_D100_30p', 'PC_D100_40p', 'PC_D100_50p', 'PC_D100_60p', 'PC_D100_70p', 'PC_D100_80p', 'PC_D100_90p',\n",
" 'PC_D100_F25_total', 'TP_D100_F50_total', 'PC_D100_L25_total', 'PC_D100_L50_total',\n",
" \n",
" 'PC_U_total', 'PC_U_avg', 'PC_U_max', 'PC_U_min', 'PC_U_medn', 'PC_U_std', \n",
" 'PC_U_10p', 'PC_U_20p', 'PC_U_30p', 'PC_U_40p', 'PC_U_50p', 'PC_U_60p', 'PC_U_70p', 'PC_U_80p', 'PC_U_90p',\n",
" 'PC_U_F25_total', 'TP_U_F50_total', 'PC_U_L25_total', 'PC_U_L50_total',\n",
" \n",
" 'PC_U100_total', 'PC_U100_avg', 'PC_U100_max', 'PC_U100_min', 'PC_U100_medn', 'PC_U100_std', \n",
" 'PC_U100_10p', 'PC_U100_20p', 'PC_U100_30p', 'PC_U100_40p', 'PC_U100_50p', 'PC_U100_60p', 'PC_U100_70p', 'PC_U100_80p', 'PC_U100_90p',\n",
" 'PC_U100_F25_total', 'TP_U100_F50_total', 'PC_U100_L25_total', 'PC_U100_L50_total',\n",
"\n",
"\n",
" 'PS_D_avg', 'PS_D_max', 'PS_D_min', 'PS_D_medn', 'PS_D_std', \n",
" 'PS_D_10p', 'PS_D_20p', 'PS_D_30p', 'PS_D_40p', 'PS_D_50p', 'PS_D_60p', 'PS_D_70p', 'PS_D_80p', 'PS_D_90p',\n",
" 'PS_D_F25_avg', 'PS_D_F50_avg', 'PS_D_L25_avg', 'PS_D_L50_avg',\n",
" \n",
" 'PS_D100_avg', 'PS_D100_max', 'PS_D100_min', 'PS_D100_medn', 'PS_D100_std', \n",
" 'PS_D100_10p', 'PS_D100_20p', 'PS_D100_30p', 'PS_D100_40p', 'PS_D100_50p', 'PS_D100_60p', 'PS_D100_70p', 'PS_D100_80p', 'PS_D100_90p',\n",
" 'PS_D100_F25_avg', 'PS_D100_F50_avg', 'PS_D100_L25_avg', 'PS_D100_L50_avg',\n",
" \n",
" 'PS_U_avg', 'PS_U_max', 'PS_U_min', 'PS_U_medn', 'PS_U_std', \n",
" 'PS_U_10p', 'PS_U_20p', 'PS_U_30p', 'PS_U_40p', 'PS_U_50p', 'PS_U_60p', 'PS_U_70p', 'PS_U_80p', 'PS_U_90p',\n",
" 'PS_U_F25_avg', 'PS_U_F50_avg', 'PS_U_L25_avg', 'PS_U_L50_avg',\n",
" \n",
" 'PS_U100_avg', 'PS_U100_max', 'PS_U100_min', 'PS_U100_medn', 'PS_U100_std', \n",
" 'PS_U100_10p', 'PS_U100_20p', 'PS_U100_30p', 'PS_U100_40p', 'PS_U100_50p', 'PS_U100_60p', 'PS_U100_70p', 'PS_U100_80p', 'PS_U100_90p',\n",
" 'PS_U100_F25_avg', 'PS_U100_F50_avg', 'PS_U100_L25_avg', 'PS_U100_L50_avg',\n",
"\n",
"\n",
" 'IAT_D_avg', 'IAT_D_max', 'IAT_D_min', 'IAT_D_medn', 'IAT_D_std', \n",
" 'IAT_D_10p', 'IAT_D_20p', 'IAT_D_30p', 'IAT_D_40p', 'IAT_D_50p', 'IAT_D_60p', 'IAT_D_70p', 'IAT_D_80p', 'IAT_D_90p',\n",
" 'IAT_D_F25_avg', 'IAT_D_F50_avg', 'IAT_D_L25_avg', 'IAT_D_L50_avg',\n",
" \n",
" 'IAT_D100_avg', 'IAT_D100_max', 'IAT_D100_min', 'IAT_D100_medn', 'IAT_D100_std', \n",
" 'IAT_D100_10p', 'IAT_D100_20p', 'IAT_D100_30p', 'IAT_D100_40p', 'IAT_D100_50p', 'IAT_D100_60p', 'IAT_D100_70p', 'IAT_D100_80p', 'IAT_D100_90p',\n",
" 'IAT_D100_F25_avg', 'IAT_D100_F50_avg', 'IAT_D100_L25_avg', 'IAT_D100_L50_avg',\n",
" \n",
"\n",
" 'IAT_U_avg', 'IAT_U_max', 'IAT_U_min', 'IAT_U_medn', 'IAT_U_std', \n",
" 'IAT_U_10p', 'IAT_U_20p', 'IAT_U_30p', 'IAT_U_40p', 'IAT_U_50p', 'IAT_U_60p', 'IAT_U_70p', 'IAT_U_80p', 'IAT_U_90p',\n",
" 'IAT_U_F25_avg', 'IAT_U_F50_avg', 'IAT_U_L25_avg', 'IAT_U_L50_avg',\n",
" \n",
" 'IAT_U100_avg', 'IAT_U100_max', 'IAT_U100_min', 'IAT_U100_medn', 'IAT_U100_std', \n",
" 'IAT_U100_10p', 'IAT_U100_20p', 'IAT_U100_30p', 'IAT_U100_40p', 'IAT_U100_50p', 'IAT_U100_60p', 'IAT_U100_70p', 'IAT_U100_80p', 'IAT_U100_90p',\n",
" 'IAT_U100_F25_avg', 'IAT_U100_F50_avg', 'IAT_U100_L25_avg', 'IAT_U100_L50_avg'\n",
"\n",
" ])\n",
"\n",
"\n",
"\n",
"iteration = 1\n",
"\n",
"mode=['5g', '4g'] \n",
"net4=['train_new_1', 'static_new_1','car_new_1','pedestrian_new_1', 'bus_new_1'] \n",
"net5=[ 'static-1','static-2', 'static-3', 'static-4', 'static-5']\n",
"\n",
"\n",
"tp_type=['tcp']\n",
"ap_type= ['http-1.1_p', 'http-1.1_np']\n",
"sv_type=['WSGI_caddy']\n",
"\n",
"\n",
"video_id=['v1','v2']\n",
"tl_scheme= ['12x4', '8x4']\n",
"vp_traces = 5 #48\n",
"vp_error= [0, 50, 100]\n",
"thread= [1]\n",
"buffer= [2,4,6]\n",
"t_segment = [60]\n",
"algo=[0,1] # adaptation algorithm FD =0 , FDB = 1\n",
"\n",
"bg_traffic = ['no'] \n",
"\n",
"host=[1] \n",
"\n",
"\n",
"for it in range (iteration):\n",
" for i in mode:\n",
" if i == '5g':\n",
" mob = net5\n",
" else:\n",
" mob = net4\n",
" for j in mob:\n",
" for k in tp_type:\n",
" for l in ap_type:\n",
" for m in sv_type:\n",
" for n in video_id:\n",
" for o in tl_scheme:\n",
" for p in range (1,vp_traces+1):\n",
" if mob == net5:\n",
" delay = 5\n",
" else:\n",
" delay = 11\n",
" for q in vp_error:\n",
" for r in thread:\n",
" for s in buffer:\n",
" for t in t_segment:\n",
" for u in algo:\n",
" for v in bg_traffic:\n",
" for w in host:\n",
" path1= str(i)+'/'+str(j)+'/'+str(k)+'/'+str(l)+'/'+str(m)+'/'+str(n)+'/'+str(o)+'/'+'VP-trace-'+str(p)+'/'+str(v)+'-bg_traffic'+'/'+'Exp'+str(it+1)+'/' \n",
" path2= 'host-'+str(w)+'_ts-'+str(t)+'_thd-'+str(r)+'_vpe-'+str(q)+'_algo-'+str(u)+'_bft-'+str(s)+'_delay-'+str(delay)+'/'\n",
" filename ='host-'+str(w)+'_ts-'+str(t)+'_thd-'+str(r)+'_vpe-'+str(q)+'_algo-'+str(u)+'_bft-'+str(s)+'_delay-'+str(delay)\n",
" path='/media/tariq/Data-House/vrexp/'+str(path1)+str(path2)+str(filename)\n",
"\n",
" try:\n",
" cols = ['Iteration', 'Type', 'Mobility', 'Delay', 'Tp_type', 'Ap_type', 'Server_type', 'V_id', 'Tile_scheme', \n",
" 'VP_trace', 'VP_error', 'Thread', 'Total_seg', 'Algo', 'Buffer_time', 'Host_num', 'Bg_traffic']\n",
" \n",
" d0 = pd.DataFrame(columns=cols)\n",
"\n",
" d0.loc[0, 'Iteration'] = it+1\n",
" d0.loc[0, 'Type'] = str(i)\n",
" d0.loc[0, 'Mobility'] = str(j)\n",
" d0.loc[0, 'Delay'] = delay\n",
" d0.loc[0, 'Tp_type'] = str(k)\n",
" d0.loc[0, 'Ap_type'] = str(l)\n",
" d0.loc[0, 'Server_type'] = str(m)\n",
" d0.loc[0, 'V_id'] = str(n)\n",
" d0.loc[0, 'Tile_scheme'] = str(o)\n",
" d0.loc[0, 'VP_trace'] = p\n",
" d0.loc[0, 'VP_error'] = q\n",
" d0.loc[0, 'Thread'] = r\n",
" d0.loc[0, 'Total_seg'] = t\n",
" d0.loc[0, 'Algo'] = u\n",
" d0.loc[0, 'Buffer_time'] = s\n",
" d0.loc[0, 'Host_num'] = w\n",
" d0.loc[0, 'Bg_traffic'] = str(v)\n",
" d0.loc[0, 'Iteration'] = str(it+1)\n",
"\n",
"\n",
" path3=str(path)+'-session1_new.csv'\n",
"\n",
" path4=str(path)+'-pcap.csv'\n",
"\n",
" d1= pd.read_csv(str(path3))\n",
"\n",
" d2= pd.read_csv(str(path4))\n",
"\n",
" d2 = d2.iloc[:,1:]\n",
"\n",
"\n",
" result = pd.concat([d0, d1, d2], axis=1)\n",
"\n",
" df=df.append(result) \n",
"\n",
" df.iloc[0:0]\n",
" print('ok')\n",
" \n",
" except:\n",
" print(path)\n",
" pass\n",
" \n",
"export_csv =df.to_csv('/media/tariq/Data-House/vrexp/new_tcp-test.csv', index=False)"
]
},
{
"cell_type": "markdown",
"id": "fca179c2",
"metadata": {},
"source": [
"### Normalize QoE values"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1429cf83",
"metadata": {},
"outputs": [],
"source": [
"df1 = pd.read_csv('/media/tariq/Data-House/vrexp/Datasets/new_tcp-test.csv')\n",
"\n",
"df1 = df1.drop(df1.index[df1['QoE'] < 0])\n",
"\n",
"a, b = 0, 5\n",
"x, y = df1.QoE.min(), df1.QoE.max()\n",
"df1['nQoE'] = (df1.QoE - x) / (y - x) * (b - a) + a\n",
"\n",
"x1, y1 = df1.QoE1.min(), df1.QoE1.max()\n",
"df1['nQoE1'] = (df1.QoE1 - x1) / (y1 - x1) * (b - a) + a\n",
"\n",
"\n",
"# x2, y2 = df1.TP_U_avg.min(), df1.TP_U_avg.max()\n",
"# df1['nTP_U_avg'] = (df1.TP_U_avg - x2) / (y2 - x2) * (b - a) + a\n",
"\n",
" \n",
"export_csv =df1.to_csv('/media/tariq/Data-House/vrexp/Datasets/new_tcp-test1.csv', index=False)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit d0fba36

Please sign in to comment.