kdd-packet-classification-ex.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_dir = os.path.expanduser('~') + '/Desktop/network/NSL-KDD/'\n",
    "train_file = data_dir + 'KDDTrain+.txt'\n",
    "test_file = data_dir + 'KDDTest+.txt'\n",
    "header_names = ['duration', 'protocol_type', 'service', 'flag', 'src_bytes', 'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot', 'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell', 'su_attempted', 'num_root', 'num_file_creations', 'num_shells', 'num_access_files', 'num_outbound_cmds', 'is_host_login', 'is_guest_login', 'count', 'srv_count', 'serror_rate', 'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', 'same_srv_rate', 'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count', 'dst_host_same_srv_rate', 'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate', 'dst_host_serror_rate', 'dst_host_srv_serror_rate', 'dst_host_rerror_rate', 'dst_host_srv_rerror_rate', 'attack_type', 'success_pred']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "category = defaultdict(list)\n",
    "category['benign'].append('normal')\n",
    "\n",
    "with open(os.path.expanduser('~') + '/Desktop/network/NSL-KDD/training_attack_types.txt', 'r') as f:\n",
    "    for line in f.readlines():\n",
    "        attack, cat = line.strip().split(' ')\n",
    "        category[cat].append(attack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "attack_mapping = dict((v,k) for k in category for v in category[k])\n",
    "print attack_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(train_file, names=header_names)\n",
    "train_df['attack_category'] = train_df['attack_type'] \\\n",
    "                                .map(lambda x: attack_mapping[x])\n",
    "train_df.drop(['success_pred'], axis=1, inplace=True)\n",
    "    \n",
    "test_df = pd.read_csv(test_file, names=header_names)\n",
    "test_df['attack_category'] = test_df['attack_type'] \\\n",
    "                                .map(lambda x: attack_mapping[x])\n",
    "test_df.drop(['success_pred'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print attack_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_attack_types = train_df['attack_type'].value_counts()\n",
    "train_attack_cats = train_df['attack_category'].value_counts()\n",
    "train_attack_types.plot(kind='barh', figsize=(20,10), fontsize=20)\n",
    "train_attack_cats.plot(kind='barh', figsize=(20,10), fontsize=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### PLOT TEST ATTACK DISTRIBUTION\n",
    "\n",
    "\n",
    "###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_Y = train_df['attack_category']\n",
    "train_x_raw = train_df.drop(['attack_category','attack_type'], axis=1)\n",
    "test_Y = test_df['attack_category']\n",
    "test_x_raw = test_df.drop(['attack_category','attack_type'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_names = defaultdict(list)\n",
    "\n",
    "with open(os.path.expanduser('~') + '/Desktop/network/NSL-KDD/kddcup.names', 'r') as f:\n",
    "    for line in f.readlines()[1:]:\n",
    "        name, nature = line.strip()[:-1].split(': ')\n",
    "        feature_names[nature].append(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "combined_df_raw = pd.concat([train_x_raw, test_x_raw])\n",
    "combined_df = pd.get_dummies(combined_df_raw, columns=feature_names['symbolic'], drop_first=True)\n",
    "\n",
    "train_x = combined_df[:len(train_x_raw)]\n",
    "test_x = combined_df[len(train_x_raw):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_x.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_x['duration'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "durations = train_x['duration'].reshape(-1, 1)\n",
    "standard_scaler = StandardScaler().fit(durations)\n",
    "scaled_durations = standard_scaler.transform(durations)\n",
    "pd.Series(scaled_durations.flatten()).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "min_max_scaler = MinMaxScaler().fit(durations)\n",
    "min_max_scaled_durations = min_max_scaler.transform(durations)\n",
    "pd.Series(min_max_scaled_durations.flatten()).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### TRY APPLYING STANDARDSCALER ON TRAIN_X and TEST_X\n",
    "\n",
    "\n",
    "###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_x.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import confusion_matrix, zero_one_loss\n",
    "\n",
    "classifier = DecisionTreeClassifier()\n",
    "classifier.fit(train_x, train_Y)\n",
    "\n",
    "pred_y = classifier.predict(test_x)\n",
    "\n",
    "results = confusion_matrix(test_Y, pred_y)\n",
    "error = zero_one_loss(test_Y, pred_y)\n",
    "\n",
    "print(results)\n",
    "print(error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE, RandomOverSampler\n",
    "from imblearn.under_sampling import ClusterCentroids\n",
    "from imblearn.under_sampling import NearMiss\n",
    "\n",
    "sm = SMOTE()\n",
    "ros = RandomOverSampler()\n",
    "cnn = ClusterCentroids()\n",
    "nm = NearMiss()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}