GAE-GAT-04-HPO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a176ddf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import glob\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import os.path as osp\n",
    "import copy\n",
    "import os\n",
    "import pickle\n",
    "from datetime import datetime\n",
    "import random\n",
    "from pm4py.objects.conversion.log import converter as log_converter\n",
    "from pm4py.objects.log.exporter.xes import exporter as xes_exporter\n",
    "from pm4py.objects.log.importer.xes import importer as xes_importer\n",
    "from pm4py.algo.filtering.log.attributes import attributes_filter\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn import metrics\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric.data import Dataset, Data\n",
    "from torch_geometric.loader import DataLoader\n",
    "from torch_geometric.nn import NNConv\n",
    "from torch_geometric.nn import GATv2Conv\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99f163b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SkinbaronDataset(Dataset):\n",
    "    def __init__(self, root, transform=None, pre_transform=None, pre_filter=None):\n",
    "        super(SkinbaronDataset, self).__init__(root, transform, pre_transform, pre_filter)\n",
    "    @property\n",
    "    def processed_file_names(self):\n",
    "        return processed_graphs\n",
    "    def len(self):\n",
    "        return len(self.processed_file_names)\n",
    "    def get(self, idx):\n",
    "        data = torch.load(osp.join(self.processed_dir, f'data_{idx}.pt'))\n",
    "        return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1706ff12",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define architecture of GAE\n",
    "class GATL1noSelf(nn.Module):\n",
    "    \"\"\"Define GATL1noSelf class for the Graph Attention neural network.(GAT)\n",
    "    with one layer, and without self loops\"\"\"\n",
    "\n",
    "    def __init__(self, size_in, size_out, edge_size, num_head, tmax, dropout_prob):\n",
    "        super(GATL1noSelf, self).__init__()\n",
    "        self.conv1 = GATv2Conv(in_channels=size_in, out_channels=size_out, heads=num_head, dropout=dropout_prob,\n",
    "                               add_self_loops=False, edge_dim=edge_size)\n",
    "        self.readout = nn.Linear(2 * num_head * size_out, tmax - 1)\n",
    "\n",
    "    def forward(self, graph):\n",
    "        graph.x = F.elu(self.conv1(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, 0])], graph.x[int(graph.edge_index[1, 0])]))\n",
    "        m_array_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "        for i in range(1, graph.edge_index.shape[1]):  # iterate over the number of edges in the batch (called graph)\n",
    "            s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, i])], graph.x[int(graph.edge_index[1, i])]))\n",
    "            edge_label_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "            m_array_pred = torch.cat((m_array_pred, edge_label_pred), 0)\n",
    "        m_array_pred = m_array_pred.view(-1, tmax - 1)\n",
    "        return m_array_pred\n",
    "\n",
    "class GATL1withSelf(nn.Module):\n",
    "    \"\"\"Define GATL1withSelf class for the Graph Attention neural network.(GAT)\n",
    "    with one layer, and self loops\"\"\"\n",
    "\n",
    "    def __init__(self, size_in, size_out, edge_size, num_head, tmax, dropout_prob):\n",
    "        super(GATL1withSelf, self).__init__()\n",
    "        self.conv1 = GATv2Conv(in_channels=size_in, out_channels=size_out, heads=num_head, edge_dim=edge_size,\n",
    "                               dropout=dropout_prob, fill_value='mean')\n",
    "        self.readout = nn.Linear(2 * num_head * size_out, tmax - 1)\n",
    "\n",
    "    def forward(self, graph):\n",
    "        graph.x = F.elu(self.conv1(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, 0])], graph.x[int(graph.edge_index[1, 0])]))\n",
    "        m_array_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "        for i in range(1, graph.edge_index.shape[1]):  # iterate over the number of edges in the batch (called graph)\n",
    "            s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, i])], graph.x[int(graph.edge_index[1, i])]))\n",
    "            edge_label_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "            m_array_pred = torch.cat((m_array_pred, edge_label_pred), 0)\n",
    "        m_array_pred = m_array_pred.view(-1, tmax - 1)\n",
    "        return m_array_pred\n",
    "    \n",
    "\n",
    "class GATL2noSelf(nn.Module):\n",
    "    \"\"\"Define GATL2noSelf class for the Graph Attention neural network.(GAT)\n",
    "    with two layers, and without self loops\"\"\"\n",
    "\n",
    "    def __init__(self, size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob):\n",
    "        super(GATL2noSelf, self).__init__()\n",
    "        self.conv1 = GATv2Conv(in_channels=size_in, out_channels=size_hid1, heads=num_head, dropout=dropout_prob,\n",
    "                               add_self_loops=False, edge_dim=edge_size)\n",
    "        self.conv2 = GATv2Conv(in_channels=num_head * size_hid1, out_channels=size_out, heads=num_head, dropout=dropout_prob,\n",
    "                               add_self_loops=False, edge_dim=edge_size)\n",
    "        self.readout = nn.Linear(2 * num_head * size_out, tmax - 1)\n",
    "\n",
    "    def forward(self, graph):\n",
    "        graph.x = F.elu(self.conv1(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        graph.x = F.elu(self.conv2(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, 0])], graph.x[int(graph.edge_index[1, 0])]))\n",
    "        m_array_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "        for i in range(1, graph.edge_index.shape[1]):  # iterate over the number of edges in the batch (called graph)\n",
    "            s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, i])], graph.x[int(graph.edge_index[1, i])]))\n",
    "            edge_label_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "            m_array_pred = torch.cat((m_array_pred, edge_label_pred), 0)\n",
    "        m_array_pred = m_array_pred.view(-1, tmax - 1)\n",
    "        return m_array_pred\n",
    "    \n",
    "class GATL2withSelf(nn.Module):\n",
    "    \"\"\"Define GATL2withSelf class for the Graph Attention neural network.(GAT)\n",
    "    with two layers, and self loops\"\"\"\n",
    "\n",
    "    def __init__(self, size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob):\n",
    "        super(GATL2withSelf, self).__init__()\n",
    "        self.conv1 = GATv2Conv(in_channels=size_in, out_channels=size_hid1, heads=num_head, edge_dim=edge_size,\n",
    "                               dropout=dropout_prob, fill_value='mean')\n",
    "        self.conv2 = GATv2Conv(in_channels=num_head * size_hid1, out_channels=size_out, heads=num_head, edge_dim=edge_size,\n",
    "                               dropout=dropout_prob, fill_value='mean')\n",
    "        self.readout = nn.Linear(2 * num_head * size_out, tmax - 1)\n",
    "\n",
    "    def forward(self, graph):\n",
    "        graph.x = F.elu(self.conv1(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        graph.x = F.elu(self.conv2(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, 0])], graph.x[int(graph.edge_index[1, 0])]))\n",
    "        m_array_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "        for i in range(1, graph.edge_index.shape[1]):  # iterate over the number of edges in the batch (called graph)\n",
    "            s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, i])], graph.x[int(graph.edge_index[1, i])]))\n",
    "            edge_label_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "            m_array_pred = torch.cat((m_array_pred, edge_label_pred), 0)\n",
    "        m_array_pred = m_array_pred.view(-1, tmax - 1)\n",
    "        return m_array_pred\n",
    "    \n",
    "class GATL3noSelf(nn.Module):\n",
    "    \"\"\"Define GATL3noSelf class for the Graph Attention neural network.(GAT)\n",
    "    with three layers, and without self loops\"\"\"\n",
    "\n",
    "    def __init__(self, size_in, size_out, size_hid1, size_hid2, edge_size, num_head, tmax, dropout_prob):\n",
    "        super(GATL3noSelf, self).__init__()\n",
    "        self.conv1 = GATv2Conv(in_channels=size_in, out_channels=size_hid1, heads=num_head, dropout=dropout_prob,\n",
    "                               add_self_loops=False, edge_dim=edge_size)\n",
    "        self.conv2 = GATv2Conv(in_channels=num_head * size_hid1, out_channels=size_hid2, heads=num_head, dropout=dropout_prob,\n",
    "                               add_self_loops=False, edge_dim=edge_size)\n",
    "        self.conv3 = GATv2Conv(in_channels=num_head * size_hid2, out_channels=size_out, heads=num_head, dropout=dropout_prob,\n",
    "                               add_self_loops=False, edge_dim=edge_size)\n",
    "        self.readout = nn.Linear(2 * num_head * size_out, tmax - 1)\n",
    "\n",
    "    def forward(self, graph):\n",
    "        graph.x = F.elu(self.conv1(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        graph.x = F.elu(self.conv2(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        graph.x = F.elu(self.conv3(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, 0])], graph.x[int(graph.edge_index[1, 0])]))\n",
    "        m_array_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "        for i in range(1, graph.edge_index.shape[1]):  # iterate over the number of edges in the batch (called graph)\n",
    "            s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, i])], graph.x[int(graph.edge_index[1, i])]))\n",
    "            edge_label_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "            m_array_pred = torch.cat((m_array_pred, edge_label_pred), 0)\n",
    "        m_array_pred = m_array_pred.view(-1, tmax - 1)\n",
    "        return m_array_pred\n",
    "    \n",
    "class GATL3withSelf(nn.Module):\n",
    "    \"\"\"Define GATL3withSelf class for the Graph Attention neural network.(GAT)\n",
    "    with three layers, and without self loops\"\"\"\n",
    "\n",
    "    def __init__(self, size_in, size_out, size_hid1, size_hid2, edge_size, num_head, tmax, dropout_prob):\n",
    "        super(GATL3withSelf, self).__init__()\n",
    "        self.conv1 = GATv2Conv(in_channels=size_in, out_channels=size_hid1, heads=num_head, edge_dim=edge_size,\n",
    "                               dropout=dropout_prob, fill_value='mean')\n",
    "        self.conv2 = GATv2Conv(in_channels=num_head * size_hid1, out_channels=size_hid2, heads=num_head, edge_dim=edge_size,\n",
    "                               dropout=dropout_prob, fill_value='mean')\n",
    "        self.conv3 = GATv2Conv(in_channels=num_head * size_hid2, out_channels=size_out, heads=num_head, edge_dim=edge_size,\n",
    "                               dropout=dropout_prob, fill_value='mean')\n",
    "        self.readout = nn.Linear(2 * num_head * size_out, tmax - 1)\n",
    "\n",
    "    def forward(self, graph):\n",
    "        graph.x = F.elu(self.conv1(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        graph.x = F.elu(self.conv2(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        graph.x = F.elu(self.conv3(graph.x, graph.edge_index, graph.edge_attr))\n",
    "        s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, 0])], graph.x[int(graph.edge_index[1, 0])]))\n",
    "        m_array_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "        for i in range(1, graph.edge_index.shape[1]):  # iterate over the number of edges in the batch (called graph)\n",
    "            s_t_nodes = torch.cat((graph.x[int(graph.edge_index[0, i])], graph.x[int(graph.edge_index[1, i])]))\n",
    "            edge_label_pred = torch.sigmoid(self.readout(s_t_nodes))\n",
    "            m_array_pred = torch.cat((m_array_pred, edge_label_pred), 0)\n",
    "        m_array_pred = m_array_pred.view(-1, tmax - 1)\n",
    "        return m_array_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "173b0198",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set the device to GPU or CPU (only if GPU is not available we will use CPU)\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "# for reproducibility of the result (note: we use SGD for optimization)\n",
    "np.random.seed(0)\n",
    "# Define binary cross entropy loss.\n",
    "batch_loss = nn.BCELoss()\n",
    "#batch_loss = nn.MSELoss() # in case we use sum of time and control-flow in m-array\n",
    "targat_path = r\"D:\\Final master thesis evaluation\\exp1\\data\"\n",
    "processed_Pattern = r\"D:\\Final master thesis evaluation\\exp1\\data\\processed\\*.pt\"\n",
    "# creating dataset\n",
    "processed_graphs = glob.glob(processed_Pattern)\n",
    "dataset = SkinbaronDataset(root=targat_path)\n",
    "# train/validation/test split\n",
    "train_dataset = dataset[0:9000]\n",
    "validation_dataset = dataset[9000:12000]\n",
    "test_dataset = dataset[12000:15000]\n",
    "print(f'Number of training graphs: {len(train_dataset)}')\n",
    "print(f'Number of validation graphs: {len(validation_dataset)}')\n",
    "print(f'Number of test graphs: {len(test_dataset)}')\n",
    "# Retrieving the case_id list from saved file on the disk:\n",
    "case_id_file = open(case_id_target, \"rb\")\n",
    "case_id_list = pickle.load(case_id_file)\n",
    "case_id_file.close()\n",
    "# find the length of the longest trace based on the edge_m_array attribute\n",
    "tmax = int(dataset.get(0).edge_m_array.shape[1]) + 1\n",
    "num_configurations = 54\n",
    "active_arms = list(range(num_configurations))\n",
    "hpo_path = r'D:\\Final master thesis evaluation\\exp2\\HPO models'\n",
    "hpo_models_target = []\n",
    "for idx in range (num_configurations):\n",
    "    hpo_models_target.append(osp.join(hpo_path, f'model_{idx}.pt'))\n",
    "first_bracket_path = r'D:\\Final master thesis evaluation\\exp2\\HPO models\\first_bracket.pkl'\n",
    "second_bracket_path = r'D:\\Final master thesis evaluation\\exp2\\HPO models\\second_bracket.pkl'\n",
    "third_bracket_path = r'D:\\Final master thesis evaluation\\exp2\\HPO models\\third_bracket.pkl'\n",
    "layer_list = []\n",
    "self_loop_list = []\n",
    "dropout_list = []\n",
    "batch_size_list = []\n",
    "bottleneck_list = []\n",
    "learning_rate_list = []\n",
    "random.seed(42)\n",
    "for j in range (num_configurations):\n",
    "    n1 = random.random()\n",
    "    if n1 > 0.5:\n",
    "        layer_list.append(2)\n",
    "    else:\n",
    "        layer_list.append(1)\n",
    "    n2 = random.random()\n",
    "    if n2 > 0.5:\n",
    "        self_loop_list.append(1)\n",
    "    else:\n",
    "        self_loop_list.append(0)\n",
    "    n3 = random.random()  \n",
    "    if n3 <= 0.33:\n",
    "        dropout_list.append(0.0)\n",
    "    elif n3 <= 0.67:\n",
    "        dropout_list.append(0.2)\n",
    "    else:\n",
    "        dropout_list.append(0.4)         \n",
    "    n4 = random.random()\n",
    "    if n4 <= 0.25:\n",
    "        batch_size_list.append(16)\n",
    "    elif n4 <= 0.5:\n",
    "        batch_size_list.append(32)\n",
    "    elif n4 <= 0.75:\n",
    "        batch_size_list.append(64)\n",
    "    else:\n",
    "        batch_size_list.append(128)\n",
    "    n5 = random.random()\n",
    "    bottleneck_list.append(2*n5+1)\n",
    "    n6 = random.random()\n",
    "    learning_rate_list.append(np.power(10, (((n6-1)/0.5)-1)))\n",
    "print(layer_list)\n",
    "print(self_loop_list)\n",
    "print(dropout_list)\n",
    "print(batch_size_list)\n",
    "print(bottleneck_list)\n",
    "print(learning_rate_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3328389d",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_id_list = case_id_list[9000:12000]\n",
    "# import label dataframe\n",
    "label_csv = pd.read_csv(r'D:\\Final master thesis evaluation\\exp1\\large-0.1-1.csv')\n",
    "label_csv.drop(\"Unnamed: 0\", axis=1, inplace=True)\n",
    "label_csv[\"case_id\"] = pd.to_numeric(label_csv[\"case_id\"])\n",
    "label_csv.case_id.astype(str).astype(int)\n",
    "label_csv.label.astype(str)\n",
    "#label_csv.dtypes\n",
    "validation_csv = label_csv.loc[label_csv['case_id'] > 9000]\n",
    "validation_csv = validation_csv.loc[label_csv['case_id'] < 12001]\n",
    "#print(len(validation_csv))\n",
    "label_csv1 = validation_csv.loc[validation_csv['label']== 'normal']\n",
    "normal_class = len(label_csv1)\n",
    "anom_class = 3000 - len(label_csv1)\n",
    "hpo_result = []\n",
    "bracket = 0\n",
    "for arm in active_arms:\n",
    "    print(arm)\n",
    "    # initialize the model\n",
    "    num_head = 1\n",
    "    size_in = dataset.get(0).x.shape[1]\n",
    "    size_out = int(bottleneck_list[arm]* dataset.get(0).x.shape[1] / num_head)\n",
    "    edge_size = dataset.get(0).edge_attr.shape[1]        \n",
    "    size_hid1 = int((bottleneck_list[arm]+1)/2 * dataset.get(0).x.shape[1] / num_head)\n",
    "    dropout_prob = dropout_list[arm]        \n",
    "    if layer_list[arm] == 1 and self_loop_list[arm] == 0:\n",
    "        model = GATL1noSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 1 and self_loop_list[arm] == 1:\n",
    "        model = GATL1withSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 2 and self_loop_list[arm] == 0:\n",
    "        model = GATL2noSelf(size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 2 and self_loop_list[arm] == 1:\n",
    "        model = GATL2withSelf(size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    # setting the minibatch size\n",
    "    batch_size = batch_size_list[arm]\n",
    "    loader = DataLoader(train_dataset, batch_size=batch_size)\n",
    "    # move to GPU (if available)\n",
    "    model = model.to(device)\n",
    "    # inizialize the optimizer\n",
    "    lr = learning_rate_list[arm]\n",
    "    weight_decay = 5e-4\n",
    "    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "    # training of the model\n",
    "    epochs = 5\n",
    "    train_hist = {}\n",
    "    train_hist['loss'] = []\n",
    "    # Initialize training\n",
    "    if bracket == 0:\n",
    "        for layer in model.children():\n",
    "            if hasattr(layer, 'reset_parameters'):\n",
    "                layer.reset_parameters()\n",
    "    model.train()\n",
    "    # training loop\n",
    "    for epoch in range(epochs):\n",
    "        for idx, data_batch in enumerate(loader):\n",
    "            data_batch = data_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_hist['loss'].append(loss.item())\n",
    "    # save and load learned parameters\n",
    "    torch.save(model.state_dict(), hpo_models_target[arm])  \n",
    "    batch_size = 1\n",
    "    loss_list = []\n",
    "    loader = DataLoader(validation_dataset, batch_size=batch_size)\n",
    "    for data_batch in loader:\n",
    "        loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "        if loss > 0:\n",
    "            loss_list.append(loss.item())\n",
    "        else:\n",
    "            loss_list.append(0)\n",
    "    GAT2_dictionary = {'case_id': validation_id_list, 'Loss': loss_list}\n",
    "    loss_df = pd.DataFrame(GAT2_dictionary)\n",
    "    loss_df[\"case_id\"] = pd.to_numeric(loss_df[\"case_id\"])\n",
    "    result = pd.merge(loss_df, validation_csv, on=[\"case_id\"])\n",
    "    sorted_result = result.sort_values(by=['Loss'], ascending = False, ignore_index = True)\n",
    "    sorted_list = sorted_result['label'].tolist()\n",
    "    predictions = []\n",
    "    for j in range (len(sorted_list)):\n",
    "        if sorted_list[j] == 'normal':\n",
    "            predictions.append(0)\n",
    "        else:\n",
    "            predictions.append(1)\n",
    "    prediction_array = np.array(predictions)\n",
    "    best_score = 0\n",
    "    for j in range (len(sorted_list)):\n",
    "        current_alarms = prediction_array[0:j+1]\n",
    "        current_normals = prediction_array[j+1:]\n",
    "        positives = j+1\n",
    "        true_positives = np.sum(current_alarms)\n",
    "        false_positives = positives - true_positives\n",
    "        negatives = 3000 - positives\n",
    "        false_negatives = np.sum(current_normals)\n",
    "        true_negatives = negatives - false_negatives\n",
    "        precision = true_positives/(true_positives+false_positives)\n",
    "        recall = true_positives/(true_positives+false_negatives)\n",
    "        f1_score = 2*precision*recall/(precision+recall)\n",
    "        if f1_score > best_score:\n",
    "            best_score = f1_score\n",
    "    hpo_result.append((arm,best_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "091583b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(hpo_result)\n",
    "hpo_file = open(first_bracket_path, \"wb\")\n",
    "pickle.dump(hpo_result, hpo_file)\n",
    "hpo_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "732e2a7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_arms = [0, 4, 7, 9, 11, 17, 19, 22, 26, 29, 31, 34, 35, 36, 37, 38, 46, 48]\n",
    "hpo_result = []\n",
    "bracket = 1\n",
    "for arm in active_arms:\n",
    "    print(arm)\n",
    "    # initialize the model\n",
    "    num_head = 1\n",
    "    size_in = dataset.get(0).x.shape[1]\n",
    "    size_out = int(bottleneck_list[arm]* dataset.get(0).x.shape[1] / num_head)\n",
    "    edge_size = dataset.get(0).edge_attr.shape[1]        \n",
    "    size_hid1 = int((bottleneck_list[arm]+1)/2 * dataset.get(0).x.shape[1] / num_head)\n",
    "    dropout_prob = dropout_list[arm]        \n",
    "    if layer_list[arm] == 1 and self_loop_list[arm] == 0:\n",
    "        model = GATL1noSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 1 and self_loop_list[arm] == 1:\n",
    "        model = GATL1withSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 2 and self_loop_list[arm] == 0:\n",
    "        model = GATL2noSelf(size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 2 and self_loop_list[arm] == 1:\n",
    "        model = GATL2withSelf(size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    # setting the minibatch size\n",
    "    batch_size = batch_size_list[arm]\n",
    "    loader = DataLoader(train_dataset, batch_size=batch_size)\n",
    "    # move to GPU (if available)\n",
    "    model = model.to(device)\n",
    "    # inizialize the optimizer\n",
    "    lr = learning_rate_list[arm]\n",
    "    weight_decay = 5e-4\n",
    "    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "    # training of the model\n",
    "    epochs = 15\n",
    "    train_hist = {}\n",
    "    train_hist['loss'] = []\n",
    "    # Initialize training\n",
    "    if bracket == 0:\n",
    "        for layer in model.children():\n",
    "            if hasattr(layer, 'reset_parameters'):\n",
    "                layer.reset_parameters()\n",
    "    model.train()\n",
    "    # training loop\n",
    "    for epoch in range(epochs):\n",
    "        for idx, data_batch in enumerate(loader):\n",
    "            data_batch = data_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_hist['loss'].append(loss.item())\n",
    "    # save and load learned parameters\n",
    "    torch.save(model.state_dict(), hpo_models_target[arm])  \n",
    "    batch_size = 1\n",
    "    loss_list = []\n",
    "    loader = DataLoader(validation_dataset, batch_size=batch_size)\n",
    "    for data_batch in loader:\n",
    "        loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "        if loss > 0:\n",
    "            loss_list.append(loss.item())\n",
    "        else:\n",
    "            loss_list.append(0)\n",
    "    GAT2_dictionary = {'case_id': validation_id_list, 'Loss': loss_list}\n",
    "    loss_df = pd.DataFrame(GAT2_dictionary)\n",
    "    loss_df[\"case_id\"] = pd.to_numeric(loss_df[\"case_id\"])\n",
    "    result = pd.merge(loss_df, validation_csv, on=[\"case_id\"])\n",
    "    sorted_result = result.sort_values(by=['Loss'], ascending = False, ignore_index = True)\n",
    "    sorted_list = sorted_result['label'].tolist()\n",
    "    predictions = []\n",
    "    for j in range (len(sorted_list)):\n",
    "        if sorted_list[j] == 'normal':\n",
    "            predictions.append(0)\n",
    "        else:\n",
    "            predictions.append(1)\n",
    "    prediction_array = np.array(predictions)\n",
    "    best_score = 0\n",
    "    for j in range (len(sorted_list)):\n",
    "        current_alarms = prediction_array[0:j+1]\n",
    "        current_normals = prediction_array[j+1:]\n",
    "        positives = j+1\n",
    "        true_positives = np.sum(current_alarms)\n",
    "        false_positives = positives - true_positives\n",
    "        negatives = 3000 - positives\n",
    "        false_negatives = np.sum(current_normals)\n",
    "        true_negatives = negatives - false_negatives\n",
    "        precision = true_positives/(true_positives+false_positives)\n",
    "        recall = true_positives/(true_positives+false_negatives)\n",
    "        f1_score = 2*precision*recall/(precision+recall)\n",
    "        if f1_score > best_score:\n",
    "            best_score = f1_score\n",
    "    hpo_result.append((arm,best_score))\n",
    "hpo_file = open(second_bracket_path, \"wb\")\n",
    "pickle.dump(hpo_result, hpo_file)\n",
    "hpo_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77325441",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(hpo_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28c1015e",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_arms = [7, 26, 36, 38, 46, 48]\n",
    "hpo_result = []\n",
    "bracket = 2\n",
    "for arm in active_arms:\n",
    "    print(arm)\n",
    "    # initialize the model\n",
    "    num_head = 1\n",
    "    size_in = dataset.get(0).x.shape[1]\n",
    "    size_out = int(bottleneck_list[arm]* dataset.get(0).x.shape[1] / num_head)\n",
    "    edge_size = dataset.get(0).edge_attr.shape[1]        \n",
    "    size_hid1 = int((bottleneck_list[arm]+1)/2 * dataset.get(0).x.shape[1] / num_head)\n",
    "    dropout_prob = dropout_list[arm]        \n",
    "    if layer_list[arm] == 1 and self_loop_list[arm] == 0:\n",
    "        model = GATL1noSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 1 and self_loop_list[arm] == 1:\n",
    "        model = GATL1withSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 2 and self_loop_list[arm] == 0:\n",
    "        model = GATL2noSelf(size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    if layer_list[arm] == 2 and self_loop_list[arm] == 1:\n",
    "        model = GATL2withSelf(size_in, size_out, size_hid1, edge_size, num_head, tmax, dropout_prob)\n",
    "        if bracket != 0:\n",
    "            model.load_state_dict(torch.load(hpo_models_target[arm]))\n",
    "    # setting the minibatch size\n",
    "    batch_size = batch_size_list[arm]\n",
    "    loader = DataLoader(train_dataset, batch_size=batch_size)\n",
    "    # move to GPU (if available)\n",
    "    model = model.to(device)\n",
    "    # inizialize the optimizer\n",
    "    lr = learning_rate_list[arm]\n",
    "    weight_decay = 5e-4\n",
    "    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "    # training of the model\n",
    "    epochs = 45\n",
    "    train_hist = {}\n",
    "    train_hist['loss'] = []\n",
    "    # Initialize training\n",
    "    if bracket == 0:\n",
    "        for layer in model.children():\n",
    "            if hasattr(layer, 'reset_parameters'):\n",
    "                layer.reset_parameters()\n",
    "    model.train()\n",
    "    # training loop\n",
    "    for epoch in range(epochs):\n",
    "        for idx, data_batch in enumerate(loader):\n",
    "            data_batch = data_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_hist['loss'].append(loss.item())\n",
    "    # save and load learned parameters\n",
    "    torch.save(model.state_dict(), hpo_models_target[arm])  \n",
    "    batch_size = 1\n",
    "    loss_list = []\n",
    "    loader = DataLoader(validation_dataset, batch_size=batch_size)\n",
    "    for data_batch in loader:\n",
    "        loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "        if loss > 0:\n",
    "            loss_list.append(loss.item())\n",
    "        else:\n",
    "            loss_list.append(0)\n",
    "    GAT2_dictionary = {'case_id': validation_id_list, 'Loss': loss_list}\n",
    "    loss_df = pd.DataFrame(GAT2_dictionary)\n",
    "    loss_df[\"case_id\"] = pd.to_numeric(loss_df[\"case_id\"])\n",
    "    result = pd.merge(loss_df, validation_csv, on=[\"case_id\"])\n",
    "    sorted_result = result.sort_values(by=['Loss'], ascending = False, ignore_index = True)\n",
    "    sorted_list = sorted_result['label'].tolist()\n",
    "    predictions = []\n",
    "    for j in range (len(sorted_list)):\n",
    "        if sorted_list[j] == 'normal':\n",
    "            predictions.append(0)\n",
    "        else:\n",
    "            predictions.append(1)\n",
    "    prediction_array = np.array(predictions)\n",
    "    best_score = 0\n",
    "    for j in range (len(sorted_list)):\n",
    "        current_alarms = prediction_array[0:j+1]\n",
    "        current_normals = prediction_array[j+1:]\n",
    "        positives = j+1\n",
    "        true_positives = np.sum(current_alarms)\n",
    "        false_positives = positives - true_positives\n",
    "        negatives = 3000 - positives\n",
    "        false_negatives = np.sum(current_normals)\n",
    "        true_negatives = negatives - false_negatives\n",
    "        precision = true_positives/(true_positives+false_positives)\n",
    "        recall = true_positives/(true_positives+false_negatives)\n",
    "        f1_score = 2*precision*recall/(precision+recall)\n",
    "        if f1_score > best_score:\n",
    "            best_score = f1_score\n",
    "    hpo_result.append((arm,best_score))\n",
    "hpo_file = open(third_bracket_path, \"wb\")\n",
    "pickle.dump(hpo_result, hpo_file)\n",
    "hpo_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ec63cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(hpo_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7845fbec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get anomaly scores produced on test dataset only for the best hyperparameter configuration\n",
    "num_head = 1\n",
    "# initialize the model\n",
    "size_in = dataset.get(0).x.shape[1]\n",
    "size_out = int(bottleneck_list[26]* dataset.get(0).x.shape[1] / num_head)\n",
    "edge_size = dataset.get(0).edge_attr.shape[1]\n",
    "model = GATL1withSelf(size_in, size_out, edge_size, num_head, tmax, dropout_prob)\n",
    "model.load_state_dict(torch.load(hpo_models_target[26]))\n",
    "# get the result in a dataframe\n",
    "batch_size = 1\n",
    "loader = DataLoader(test_dataset, batch_size=batch_size)\n",
    "test_id_list = case_id_list[12000:15000]\n",
    "loss_list = []\n",
    "for data_batch in loader:\n",
    "    loss = batch_loss(model.forward(data_batch).float(), data_batch.edge_m_array.float())\n",
    "    if loss > 0:\n",
    "        loss_list.append(loss.item())\n",
    "    else:\n",
    "        loss_list.append(0)  \n",
    "GAT2_dictionary = {'case_id': test_id_list, 'Loss': loss_list}\n",
    "loss_df = pd.DataFrame(GAT2_dictionary)\n",
    "loss_df.to_pickle(r'D:\\Final master thesis evaluation\\exp2\\bestgatmodl_test_loss.pt')\n",
    "loss_df[\"case_id\"] = pd.to_numeric(loss_df[\"case_id\"])\n",
    "result = pd.merge(loss_df, test_csv, on=[\"case_id\"])\n",
    "sorted_result = result.sort_values(by=['Loss'], ascending = False, ignore_index = True)\n",
    "sorted_list = sorted_result['label'].tolist()\n",
    "predictions = []\n",
    "for j in range (len(sorted_list)):\n",
    "    if sorted_list[j] == 'normal':\n",
    "        predictions.append(0)\n",
    "    else:\n",
    "        predictions.append(1)\n",
    "prediction_array = np.array(predictions)\n",
    "best_score = 0\n",
    "for j in range (len(sorted_list)):\n",
    "    current_alarms = prediction_array[0:j+1]\n",
    "    current_normals = prediction_array[j+1:]\n",
    "    positives = j+1\n",
    "    true_positives = np.sum(current_alarms)\n",
    "    false_positives = positives - true_positives\n",
    "    negatives = 3000 - positives\n",
    "    false_negatives = np.sum(current_normals)\n",
    "    true_negatives = negatives - false_negatives\n",
    "    precision = true_positives/(true_positives+false_positives)\n",
    "    recall = true_positives/(true_positives+false_negatives)\n",
    "    f1_score = 2*precision*recall/(precision+recall)\n",
    "    if f1_score > best_score:\n",
    "        best_score = f1_score\n",
    "        precision_on_normal = true_negatives/(true_negatives+false_negatives)\n",
    "        recall_on_normal = true_negatives/(true_negatives+false_positives)\n",
    "        f1_score_on_normal = 2*precision_on_normal*recall_on_normal/(precision_on_normal+recall_on_normal)\n",
    "        macro_f1_score = (f1_score + f1_score_on_normal)/2\n",
    "        best_cut = [j, precision, recall, f1_score, precision_on_normal, recall_on_normal,\n",
    "                    f1_score_on_normal, macro_f1_score] \n",
    "        fp_for_best = false_positives\n",
    "print(best_cut, fp_for_best)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}