3_ReadPCAP_ExtractFeatures.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/alirezashirmarz/XR-AR_NTC/blob/main/3_ReadPCAP_ExtractFeatures.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O9DFHI6cqASV"
      },
      "source": [
        "# **Title:** ***Extract Features (IPI, FS, IFI) from the PCAP File***\n",
        "    ##Pay attention to configuration in each cell"
      ]
    },
    {
      "cell_type": "raw",
      "metadata": {
        "id": "QVNiZLKiqASW"
      },
      "source": [
        "# install prerequirements"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AGDPAnkIqASW"
      },
      "outputs": [],
      "source": [
        "!pip install scapy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r92sWqcXqASX"
      },
      "source": [
        "# 1- **Find the Files in the root directory e.g. pcap or pcapnj files**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bUNTzYKaqASX"
      },
      "outputs": [],
      "source": [
        "# The root Directory (Enter your address)\n",
        "root_directory = r'Directory includes PCAP or PCAPnj Files'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jNFSj8tlqASX"
      },
      "outputs": [],
      "source": [
        "# Find all pcap files in  a directory\n",
        "import os\n",
        "import glob\n",
        "\n",
        "def find_files_with_extension(root_dir, extension):\n",
        "  ''' Function: Find the files with sxpecific extension\n",
        "  Output: list of files full address '''\n",
        "    file_list = []\n",
        "    for root, dirs, files in os.walk(root_dir):\n",
        "        for file in files:\n",
        "            if file.endswith(extension):\n",
        "                file_list.append(os.path.join(root, file))\n",
        "    return file_list\n",
        "\n",
        "# set the extension to find from the rood directory\n",
        "extension = '.pcapng'  # '.pcap' or '.pcapng'\n",
        "files_with_extension = find_files_with_extension(root_directory, extension)\n",
        "\n",
        "# print to show the found files\n",
        "print(\"Files found:\")\n",
        "for file_path in files_with_extension:\n",
        "    print(file_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7OuA7z26qASZ"
      },
      "source": [
        "# 2- **Extract the Features (IPI,IFI, FS, & PS) of flow from PCAP(NJ)**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x3Acg_2gqASb"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "import pandas as pd\n",
        "import scapy.all as scapy\n",
        "from collections import defaultdict\n",
        "\n",
        "def process_pcap(pcap_file, csv_file):\n",
        "  ''' Extrct the features from the PCAP(NJ) files\n",
        "  Input: the Pcap files address and the address of csv files to be stored\n",
        "  Output: the csv file stored in the csv file address'''\n",
        "    packets = scapy.rdpcap(pcap_file)\n",
        "    flows = defaultdict(list)\n",
        "\n",
        "    # Process packets and organize them into TCP/UDP/QUIC flows\n",
        "    # Process packets and organize them into TCP/UDP/QUIC flows\n",
        "    for packet in packets:\n",
        "        protocol = None\n",
        "        if packet.haslayer(scapy.TCP):\n",
        "            protocol = 'TCP'\n",
        "            ip_layer = packet[scapy.IP] if packet.haslayer(scapy.IP) else packet[scapy.IPv6]\n",
        "            transport_layer = packet[scapy.TCP]\n",
        "        elif packet.haslayer(scapy.UDP):\n",
        "            protocol = 'UDP'\n",
        "            ip_layer = packet[scapy.IP] if packet.haslayer(scapy.IP) else packet[scapy.IPv6]\n",
        "            transport_layer = packet[scapy.UDP]\n",
        "        elif \"quic\" in str(packet).lower():  # Heuristic for QUIC\n",
        "            protocol = 'QUIC'\n",
        "            ip_layer = packet[scapy.IP] if packet.haslayer(scapy.IP) else packet[scapy.IPv6]\n",
        "            transport_layer = packet[scapy.UDP]  # QUIC runs over UDP\n",
        "\n",
        "        if protocol:\n",
        "            flow_key = (\n",
        "                ip_layer.src, ip_layer.dst,\n",
        "                transport_layer.sport, transport_layer.dport, ip_layer.name, protocol\n",
        "            )\n",
        "            flows[flow_key].append(packet)\n",
        "\n",
        "\n",
        "    with open(csv_file, 'w', newline='') as file:\n",
        "        writer = csv.writer(file)\n",
        "        writer.writerow([\n",
        "            'ID', 'SrcIP', 'DstIP', 'SrcPort', 'DstPort',\n",
        "            'IPVersion', 'Protocol', 'PS', 'IPI', 'FlowSizeBytes',\n",
        "            'FlowSizePackets', 'FS', 'FS(PKT)',\n",
        "            'NumFrames', 'IFI'\n",
        "        ])\n",
        "\n",
        "        for i, (flow_key, packets) in enumerate(flows.items()):\n",
        "            src_ip, dst_ip, src_port, dst_port, ip_version, protocol = flow_key\n",
        "\n",
        "            timestamps = [packet.time for packet in packets]\n",
        "            ip_intervals = [j - i for i, j in zip(timestamps[:-1], timestamps[1:])]\n",
        "            packet_sizes = [len(packet) for packet in packets]\n",
        "            total_flow_bytes = sum(packet_sizes)\n",
        "            total_flow_packets = len(packets)\n",
        "\n",
        "            avg_packet_size = total_flow_bytes / total_flow_packets if total_flow_packets else 0\n",
        "            avg_ipi = sum(ip_intervals) / len(ip_intervals) if ip_intervals else 0\n",
        "\n",
        "            # Frame Analysis (specific to UDP and TCP, might not be accurate for QUIC)\n",
        "            frame_sizes_bytes, frame_sizes_packets, frame_intervals = analyze_frames(packets, protocol)\n",
        "\n",
        "            avg_frame_size_bytes = sum(frame_sizes_bytes) / len(frame_sizes_bytes) if frame_sizes_bytes else 0\n",
        "            avg_frame_size_packets = sum(frame_sizes_packets) / len(frame_sizes_packets) if frame_sizes_packets else 0\n",
        "            num_frames = len(frame_sizes_bytes)\n",
        "            avg_ifi = sum(frame_intervals) / len(frame_intervals) if frame_intervals else 0\n",
        "\n",
        "            # Writing to CSV\n",
        "            writer.writerow([\n",
        "                i + 1, src_ip, dst_ip, src_port, dst_port, ip_version, protocol,\n",
        "                avg_packet_size, avg_ipi, total_flow_bytes, total_flow_packets,\n",
        "                avg_frame_size_bytes, avg_frame_size_packets, num_frames, avg_ifi\n",
        "            ])\n",
        "\n",
        "def analyze_frames(packets, protocol):\n",
        "  '''Function: Analyze the frame\n",
        "  Input: packets & Protocol\n",
        "  Output: frame_sizes_bytes, frame_sizes_packets, frame_intervals'''\n",
        "    frame_sizes_bytes = []\n",
        "    frame_sizes_packets = []\n",
        "    frame_intervals = []\n",
        "    current_frame_size_bytes = 0\n",
        "    current_frame_size_packets = 0\n",
        "    current_frame_start_time = None\n",
        "    last_length = None\n",
        "\n",
        "    for packet in packets:\n",
        "        if packet.haslayer(scapy.TCP) or packet.haslayer(scapy.UDP):\n",
        "            if protocol == 'TCP':\n",
        "                # For TCP, calculate the payload length\n",
        "                if packet.haslayer(scapy.Raw):\n",
        "                    length = len(packet[scapy.Raw].load)\n",
        "                else:\n",
        "                    length = 0\n",
        "            else:\n",
        "                # For UDP, use the length attribute\n",
        "                length = packet[scapy.UDP].len\n",
        "\n",
        "            if current_frame_start_time is None:\n",
        "                current_frame_start_time = packet.time\n",
        "\n",
        "            if last_length is not None and length != last_length:\n",
        "                frame_sizes_bytes.append(current_frame_size_bytes)\n",
        "                frame_sizes_packets.append(current_frame_size_packets)\n",
        "                frame_intervals.append(packet.time - current_frame_start_time)\n",
        "                current_frame_size_bytes = 0\n",
        "                current_frame_size_packets = 0\n",
        "                current_frame_start_time = packet.time\n",
        "\n",
        "            current_frame_size_bytes += len(packet)\n",
        "            current_frame_size_packets += 1\n",
        "            last_length = length\n",
        "\n",
        "    if current_frame_size_bytes > 0:\n",
        "        frame_sizes_bytes.append(current_frame_size_bytes)\n",
        "        frame_sizes_packets.append(current_frame_size_packets)\n",
        "\n",
        "    return frame_sizes_bytes, frame_sizes_packets, frame_intervals\n",
        "\n",
        "# Main function to run and extract the features and stor in csv format\n",
        "def main(pcap_file, csv_file):\n",
        "    process_pcap(pcap_file, csv_file + '.csv')\n",
        "\n",
        "# Execute for all files found!\n",
        "if __name__ == \"__main__\":\n",
        "    for addr in files_with_extension:\n",
        "        i += 1\n",
        "        csv_path = addr + '.csv'\n",
        "        main(addr, csv_path)\n",
        "        print(f'The dataset {i}: {csv_path} was stored!')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qZDPoalFqASa"
      },
      "source": [
        "# 3- **Read the CSV and Merge Them**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CsL1ke4LqASa"
      },
      "outputs": [],
      "source": [
        "# Read and merge the csv files\n",
        "\n",
        "'''\n",
        "# Configuration\n",
        "    # Input:\n",
        "      1- root-directory--> It is the directory with csv files (In this case, it\n",
        "      uses the roo directory from 2 previous cell)\n",
        "      2- ds_name --> the name you want to save the csv file (without extension)\n",
        "    # Output:\n",
        "      1- Save the\n",
        "'''\n",
        "\n",
        "#The name of merded dataset (default is DS)\"\n",
        "ds_name= 'DS'\n",
        "print(root_directory)\n",
        "csvfiles = find_files_with_extension(root_directory, 'csv')\n",
        "dff = []\n",
        "for mycsv in csvfiles:\n",
        "    mydf = pd.read_csv(mycsv)\n",
        "    dff.append(mydf)\n",
        "df = pd.concat(dff, ignore_index=True, sort=False)\n",
        "#df = pd.concat(dff, ignore_index=True)\n",
        "df.to_csv(root_directory +'/' + ds_name +'.csv', index=False)\n",
        "#df.to_csv(root_directory + '\\\\AR.csv', index=False)\n",
        "print(f\"The df with {df.shape[0]} rows and {df.shape[1]} columns was stored as csv file!\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f6-OyAilqASa"
      },
      "source": [
        "# 4- **Pre-Processing**\n",
        "## (4-1) Add direction for each flow & (4-2) remove the NAN Values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ozs9Zw1MqASa"
      },
      "outputs": [],
      "source": [
        "mycsvfile= root_directory + '/DS.csv'\n",
        "Directionfiles = mycsvfile.split('.')[0]+'2'+'.csv'\n",
        "Removedfile = mycsvfile.split('.')[0]+'3'+'.csv'"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **4-1- add the direction to the CSV file**"
      ],
      "metadata": {
        "id": "nzGLqGUSsjd5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x0RymoqOqASa"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "\n",
        "def add_direction_feature(input_file, output_file):\n",
        "    with open(input_file, mode='r', newline='') as infile:\n",
        "        reader = csv.DictReader(infile)\n",
        "        rows = list(reader)\n",
        "\n",
        "    # Keep track of seen destination IPs and ports\n",
        "    seen_destinations = set()\n",
        "\n",
        "    for row in rows:\n",
        "        src_tuple = (row['SrcIP'], row['SrcPort'])\n",
        "        dst_tuple = (row['DstIP'], row['DstPort'])\n",
        "\n",
        "        # Determine direction\n",
        "        if src_tuple in seen_destinations:\n",
        "            row['Direction'] = 'Downlink'\n",
        "        else:\n",
        "            row['Direction'] = 'Uplink'\n",
        "            seen_destinations.add(dst_tuple)\n",
        "\n",
        "    # Write to new CSV file\n",
        "    with open(output_file, mode='w', newline='') as outfile:\n",
        "        writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames + ['Direction'])\n",
        "        writer.writeheader()\n",
        "        writer.writerows(rows)\n",
        "\n",
        "# Example usage\n",
        "add_direction_feature(mycsvfile,Directionfiles)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **4-2- Remove the flows without Frames**"
      ],
      "metadata": {
        "id": "Nehj-vpAs9Mh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uhZnsRTCqASb"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "\n",
        "def filter_csv(input_file, output_file):\n",
        "    with open(input_file, mode='r', newline='') as infile, open(output_file, mode='w', newline='') as outfile:\n",
        "        reader = csv.DictReader(infile)\n",
        "        writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames)\n",
        "\n",
        "        writer.writeheader()\n",
        "\n",
        "        for row in reader:\n",
        "            num_frames = int(row['NumFrames'])\n",
        "            ifi = float(row['IFI'])\n",
        "\n",
        "            # Keep the row if NumFrames is greater than 1 and IFI is not 0\n",
        "            if num_frames >= 1 and ifi != 0:\n",
        "                writer.writerow(row)\n",
        "\n",
        "# Example usage\n",
        "filter_csv(Directionfiles, Removedfile)\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}