train_meta_learner_classification.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing and Preparing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fps_blend_comp</th>\n",
       "      <th>bdid</th>\n",
       "      <th>c1</th>\n",
       "      <th>prop</th>\n",
       "      <th>val_1</th>\n",
       "      <th>val_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1.2037544532748699, 1.852957906712173, 1.1290...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.8000</td>\n",
       "      <td>Tg</td>\n",
       "      <td>238.15</td>\n",
       "      <td>238.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1.2640287769784173, 1.6689081011847582, 1.076...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>Tg</td>\n",
       "      <td>248.15</td>\n",
       "      <td>248.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[1.223196881091618, 1.7872573897537893, 1.1111...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.7000</td>\n",
       "      <td>Tg</td>\n",
       "      <td>249.15</td>\n",
       "      <td>249.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[1.2640287769784173, 1.6689081011847582, 1.076...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>Tg</td>\n",
       "      <td>338.15</td>\n",
       "      <td>338.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1.2037544532748699, 1.852957906712173, 1.1290...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.8000</td>\n",
       "      <td>Tg</td>\n",
       "      <td>342.15</td>\n",
       "      <td>342.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4505</th>\n",
       "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.032...</td>\n",
       "      <td>2045</td>\n",
       "      <td>0.7500</td>\n",
       "      <td>Td</td>\n",
       "      <td>643.15</td>\n",
       "      <td>643.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4506</th>\n",
       "      <td>[1.2449499208854404, 1.5672387794174951, 1.0, ...</td>\n",
       "      <td>2051</td>\n",
       "      <td>0.4970</td>\n",
       "      <td>Td</td>\n",
       "      <td>629.15</td>\n",
       "      <td>629.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4507</th>\n",
       "      <td>[1.244969919920322, 1.567610622468521, 1.0, 1....</td>\n",
       "      <td>2051</td>\n",
       "      <td>0.4990</td>\n",
       "      <td>Td</td>\n",
       "      <td>630.65</td>\n",
       "      <td>630.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4508</th>\n",
       "      <td>[1.244959920322565, 1.5674246788897548, 1.0, 1...</td>\n",
       "      <td>2051</td>\n",
       "      <td>0.4980</td>\n",
       "      <td>Td</td>\n",
       "      <td>631.15</td>\n",
       "      <td>631.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4509</th>\n",
       "      <td>[1.4256154269334338, 2.1477183732292877, 1.0, ...</td>\n",
       "      <td>2100</td>\n",
       "      <td>0.9375</td>\n",
       "      <td>Td</td>\n",
       "      <td>597.15</td>\n",
       "      <td>597.15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4510 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         fps_blend_comp  bdid      c1 prop  \\\n",
       "0     [1.2037544532748699, 1.852957906712173, 1.1290...   547  0.8000   Tg   \n",
       "1     [1.2640287769784173, 1.6689081011847582, 1.076...   547  0.5000   Tg   \n",
       "2     [1.223196881091618, 1.7872573897537893, 1.1111...   547  0.7000   Tg   \n",
       "3     [1.2640287769784173, 1.6689081011847582, 1.076...   547  0.5000   Tg   \n",
       "4     [1.2037544532748699, 1.852957906712173, 1.1290...   547  0.8000   Tg   \n",
       "...                                                 ...   ...     ...  ...   \n",
       "4505  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.032...  2045  0.7500   Td   \n",
       "4506  [1.2449499208854404, 1.5672387794174951, 1.0, ...  2051  0.4970   Td   \n",
       "4507  [1.244969919920322, 1.567610622468521, 1.0, 1....  2051  0.4990   Td   \n",
       "4508  [1.244959920322565, 1.5674246788897548, 1.0, 1...  2051  0.4980   Td   \n",
       "4509  [1.4256154269334338, 2.1477183732292877, 1.0, ...  2100  0.9375   Td   \n",
       "\n",
       "       val_1   val_2  \n",
       "0     238.15  238.15  \n",
       "1     248.15  248.15  \n",
       "2     249.15  249.15  \n",
       "3     338.15  338.15  \n",
       "4     342.15  342.15  \n",
       "...      ...     ...  \n",
       "4505  643.15  643.15  \n",
       "4506  629.15  629.15  \n",
       "4507  630.65  630.65  \n",
       "4508  631.15  631.15  \n",
       "4509  597.15  597.15  \n",
       "\n",
       "[4510 rows x 6 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# df = pd.read_pickle('T_combined_final_double_selector.pkl')\n",
    "df = pd.read_pickle('classification_dataset.pkl')\n",
    "df.dropna(inplace=True)\n",
    "df.reset_index(drop=True, inplace = True)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "df['fps_blend_comp'] = (np.vstack(df.fps_blend_comp) - 1).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['fps_blend_comp'] = (np.around(np.vstack(df.fps_blend_comp),decimals=5 )).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['miscibility'] = (df.val_1 == df.val_2)*1 # Singled value = 1 'miscible'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['prop'] != 'Td']\n",
    "df.insert(0, 'selector', pd.get_dummies(df.prop).values.tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "  try:\n",
    "    # Currently, memory growth needs to be the same across GPUs\n",
    "    for gpu in gpus:\n",
    "      tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "  except RuntimeError as e:\n",
    "    # Memory growth must be set before GPUs have been initialized\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(drop = True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>selector</th>\n",
       "      <th>fps_blend_comp</th>\n",
       "      <th>bdid</th>\n",
       "      <th>c1</th>\n",
       "      <th>prop</th>\n",
       "      <th>val_1</th>\n",
       "      <th>val_2</th>\n",
       "      <th>miscibility</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>[0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.80</td>\n",
       "      <td>Tg</td>\n",
       "      <td>238.15</td>\n",
       "      <td>238.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>[0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.50</td>\n",
       "      <td>Tg</td>\n",
       "      <td>248.15</td>\n",
       "      <td>248.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>[0.2232, 0.78726, 0.11111, 0.24795, 0.21212, 0...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.70</td>\n",
       "      <td>Tg</td>\n",
       "      <td>249.15</td>\n",
       "      <td>249.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>[0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.50</td>\n",
       "      <td>Tg</td>\n",
       "      <td>338.15</td>\n",
       "      <td>338.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>[0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...</td>\n",
       "      <td>547</td>\n",
       "      <td>0.80</td>\n",
       "      <td>Tg</td>\n",
       "      <td>342.15</td>\n",
       "      <td>342.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4244</th>\n",
       "      <td>[0, 1]</td>\n",
       "      <td>[0.24404, 0.5775, 0.0, 0.07748, 0.0, 0.7198, 0...</td>\n",
       "      <td>2151</td>\n",
       "      <td>0.28</td>\n",
       "      <td>Tm</td>\n",
       "      <td>421.75</td>\n",
       "      <td>421.75</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4245</th>\n",
       "      <td>[0, 1]</td>\n",
       "      <td>[0.18245, 0.58736, 0.0, 0.23073, 0.0, 0.5371, ...</td>\n",
       "      <td>2151</td>\n",
       "      <td>0.73</td>\n",
       "      <td>Tm</td>\n",
       "      <td>422.65</td>\n",
       "      <td>422.65</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4246</th>\n",
       "      <td>[0, 1]</td>\n",
       "      <td>[0.20499, 0.58362, 0.0, 0.16797, 0.0, 0.60137,...</td>\n",
       "      <td>2151</td>\n",
       "      <td>0.56</td>\n",
       "      <td>Tm</td>\n",
       "      <td>422.75</td>\n",
       "      <td>422.75</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4247</th>\n",
       "      <td>[0, 1]</td>\n",
       "      <td>[0.21712, 0.60635, 0.0, 0.14943, 0.00689, 0.63...</td>\n",
       "      <td>2152</td>\n",
       "      <td>0.52</td>\n",
       "      <td>Tm</td>\n",
       "      <td>424.25</td>\n",
       "      <td>424.25</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4248</th>\n",
       "      <td>[0, 1]</td>\n",
       "      <td>[0.24132, 0.59341, 0.0, 0.08992, 0.00436, 0.71...</td>\n",
       "      <td>2152</td>\n",
       "      <td>0.33</td>\n",
       "      <td>Tm</td>\n",
       "      <td>425.25</td>\n",
       "      <td>425.25</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4249 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     selector                                     fps_blend_comp  bdid    c1  \\\n",
       "0      [1, 0]  [0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...   547  0.80   \n",
       "1      [1, 0]  [0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...   547  0.50   \n",
       "2      [1, 0]  [0.2232, 0.78726, 0.11111, 0.24795, 0.21212, 0...   547  0.70   \n",
       "3      [1, 0]  [0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...   547  0.50   \n",
       "4      [1, 0]  [0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...   547  0.80   \n",
       "...       ...                                                ...   ...   ...   \n",
       "4244   [0, 1]  [0.24404, 0.5775, 0.0, 0.07748, 0.0, 0.7198, 0...  2151  0.28   \n",
       "4245   [0, 1]  [0.18245, 0.58736, 0.0, 0.23073, 0.0, 0.5371, ...  2151  0.73   \n",
       "4246   [0, 1]  [0.20499, 0.58362, 0.0, 0.16797, 0.0, 0.60137,...  2151  0.56   \n",
       "4247   [0, 1]  [0.21712, 0.60635, 0.0, 0.14943, 0.00689, 0.63...  2152  0.52   \n",
       "4248   [0, 1]  [0.24132, 0.59341, 0.0, 0.08992, 0.00436, 0.71...  2152  0.33   \n",
       "\n",
       "     prop   val_1   val_2  miscibility  \n",
       "0      Tg  238.15  238.15            1  \n",
       "1      Tg  248.15  248.15            1  \n",
       "2      Tg  249.15  249.15            1  \n",
       "3      Tg  338.15  338.15            1  \n",
       "4      Tg  342.15  342.15            1  \n",
       "...   ...     ...     ...          ...  \n",
       "4244   Tm  421.75  421.75            1  \n",
       "4245   Tm  422.65  422.65            1  \n",
       "4246   Tm  422.75  422.75            1  \n",
       "4247   Tm  424.25  424.25            1  \n",
       "4248   Tm  425.25  425.25            1  \n",
       "\n",
       "[4249 rows x 8 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Converting the dataset for training the model using tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import QuantileTransformer, MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "tf.random.set_seed(123)\n",
    "\n",
    "\n",
    "df = df.sample(frac=1, random_state=123)\n",
    "\n",
    "target_property_scaler = MinMaxScaler\n",
    "\n",
    "batch_size = 200\n",
    "\n",
    "skf = StratifiedKFold(n_splits=5)\n",
    "\n",
    "props = df.prop.unique().tolist()\n",
    "\n",
    "datasets = []\n",
    "def get_dataset(df):\n",
    "    fps_blends = np.stack(df.fps_blend_comp.values).astype(np.float32)\n",
    "\n",
    "    selector = np.stack(df.selector.values).astype(np.float32)\n",
    "    target = df.miscibility.values.astype(int)[:, np.newaxis]\n",
    "    bdid = np.stack(df.bdid.values).astype(np.str)[:, np.newaxis]\n",
    "    # miscibility = df.miscibility.values.astype(int)[:, np.newaxis]\n",
    "    c1 = df.c1.values.astype(np.float32)[:, np.newaxis]\n",
    "    # type_poly = np.stack(df.type.values).astype(np.str)[:, np.newaxis]\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(({'selector': selector, 'fps_blends': fps_blends, 'prop': df.prop, 'c1': c1, 'bdid':bdid}, target))\n",
    "\n",
    "    dataset = dataset.cache().batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\n",
    "    return dataset\n",
    "\n",
    "\n",
    "training_df, test_df = train_test_split(df, test_size=0.2, stratify=df.prop, random_state=123)\n",
    "training_df, test_df = training_df.copy(), test_df.copy()\n",
    "\n",
    "for train_index, val_index in skf.split(training_df, training_df.prop):\n",
    "    \n",
    "    train_df = df.iloc[train_index].copy()\n",
    "    val_df = df.iloc[val_index].copy()\n",
    "    \n",
    "    \n",
    "    datasets.append({'train': get_dataset(train_df), 'val': get_dataset(val_df)})\n",
    "datasets_final = {'train': get_dataset(training_df), 'test': get_dataset(test_df)}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': <PrefetchDataset element_spec=({'selector': TensorSpec(shape=(None, 2), dtype=tf.float32, name=None), 'fps_blends': TensorSpec(shape=(None, 872), dtype=tf.float32, name=None), 'prop': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'c1': TensorSpec(shape=(None, 1), dtype=tf.float32, name=None), 'bdid': TensorSpec(shape=(None, 1), dtype=tf.string, name=None)}, TensorSpec(shape=(None, 1), dtype=tf.int64, name=None))>,\n",
       " 'test': <PrefetchDataset element_spec=({'selector': TensorSpec(shape=(None, 2), dtype=tf.float32, name=None), 'fps_blends': TensorSpec(shape=(None, 872), dtype=tf.float32, name=None), 'prop': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'c1': TensorSpec(shape=(None, 1), dtype=tf.float32, name=None), 'bdid': TensorSpec(shape=(None, 1), dtype=tf.string, name=None)}, TensorSpec(shape=(None, 1), dtype=tf.int64, name=None))>}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_final"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow.keras as tfk\n",
    "import tensorflow as tf\n",
    "from datetime import datetime \n",
    "from tensorflow.python.keras.engine import data_adapter\n",
    "from keras_tuner import HyperModel\n",
    "from keras_tuner.tuners import Hyperband, RandomSearch\n",
    "import tensorflow_addons as tfa\n",
    "from tensorflow import keras\n",
    "\n",
    "df.fps_blend_comp.to_pickle('pkl/fingerprints_combined_classification_final.pkl')\n",
    "class MetaModel(tfk.Model):\n",
    "    def __init__(self, hp): #, scale):\n",
    "        super().__init__()\n",
    "        self.base_models = []\n",
    "        # self.scaler_type = scale\n",
    "        for num in range(5):\n",
    "            model = tf.keras.models.load_model(f'models/cv_final_class/{num}')\n",
    "            model.trainable = False\n",
    "            self.base_models.append(model)\n",
    "        \n",
    "        self.my_layers = []\n",
    "        \n",
    "        for i in range(hp.Int('num_layers', 1, 2)): \n",
    "            new_step = [               \n",
    "            tf.keras.layers.Dense(units=hp.Int('units_' + str(i),\n",
    "                                            min_value=64,\n",
    "                                            max_value=544,\n",
    "                                            step=64),),\n",
    "            \n",
    "            tf.keras.layers.PReLU(),\n",
    "            tf.keras.layers.Dropout(hp.Float(\n",
    "                'dropout_' + str(i),\n",
    "                min_value=0.0,\n",
    "                max_value=0.5,\n",
    "                default=0.25,\n",
    "                step=0.05,\n",
    "            )),]\n",
    "            self.my_layers.append(new_step)\n",
    "            \n",
    "        self.my_layers.append([tf.keras.layers.Dense(1, activation = 'sigmoid')])\n",
    "\n",
    "    def call(self, inputs, training=None): \n",
    "        \n",
    "        # drop prop\n",
    "        if 'prop' in inputs:\n",
    "            del inputs['prop']\n",
    "        x = []\n",
    "        for base in self.base_models:\n",
    "            if training:\n",
    "                res = base.call(inputs, training)\n",
    "            else:\n",
    "                res = base.call(inputs)\n",
    "            x.append(res)\n",
    "        x = tf.concat(x, -1)\n",
    "        \n",
    "        for num, layer_step in enumerate(self.my_layers):\n",
    "            for layer in layer_step:\n",
    "                x = layer(x)\n",
    "\n",
    "        return x\n",
    "    \n",
    "    def predict_step(self, data):\n",
    "        data = data_adapter.expand_1d(data)\n",
    "        x, _, _ = data_adapter.unpack_x_y_sample_weight(data)\n",
    "        # drop prop here\n",
    "        prop = x['prop']\n",
    "        # bdid = x['bdid']\n",
    "        # type_poly = x['type']\n",
    "        selector = x['selector']\n",
    "        bdid = x['bdid']\n",
    "        c1 = x['c1']\n",
    "        # miscibility = x['miscibility']\n",
    "        del x['prop']\n",
    "        # del x['bdid']\n",
    "        # del x['selector']\n",
    "        return self(x, training=True), data[-1], prop, bdid, c1, selector\n",
    "\n",
    "    \n",
    "    @tf.function(input_signature=[])\n",
    "    def fp_order(self):\n",
    "        fp_order = tf.convert_to_tensor(list(pd.read_pickle('pkl/fingerprints_combined_classification_final.pkl').columns))\n",
    "        return fp_order\n",
    "        \n",
    "    \n",
    "def build_model(hp):\n",
    "    model = MetaModel(hp)\n",
    "    opt = tf.keras.optimizers.Adam(\n",
    "            hp.Choice('learning_rate',\n",
    "                      values=[1e-3]))\n",
    "    opt = tfa.optimizers.SWA(opt)\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=opt,\n",
    "        loss=keras.losses.BinaryCrossentropy(from_logits=False),)\n",
    "    return model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import IPython\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "results, property_metric = [], []\n",
    "prec_Tg = []\n",
    "recall_Tg = []\n",
    "accuracy_Tg = []\n",
    "prec_Tm = []\n",
    "recall_Tm = []\n",
    "accuracy_Tm = []\n",
    "\n",
    "\n",
    "tuner = Hyperband(\n",
    "    build_model,\n",
    "    objective='val_loss',\n",
    "    max_epochs=300,\n",
    "    # hyperband_iterations=3,\n",
    "    seed=10,\n",
    "    directory=f'hyperparameter_search_meta_learner_mt_class_final',\n",
    "    project_name='fold_0',\n",
    "    )\n",
    "\n",
    "reduce_lr = tfk.callbacks.ReduceLROnPlateau(\n",
    "    factor=0.9,\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    ")\n",
    "\n",
    "earlystop = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=40)\n",
    "\n",
    "class ClearTrainingOutput(tf.keras.callbacks.Callback):\n",
    "    def on_train_end(*args, **kwargs):\n",
    "        IPython.display.clear_output(wait = True)\n",
    "\n",
    "# Create an instance of the model\n",
    "tuner.search(datasets_final['test'],\n",
    "            epochs=300,\n",
    "            validation_data=datasets_final['train'],\n",
    "            callbacks=[earlystop, reduce_lr, ClearTrainingOutput()],\n",
    "            verbose=0\n",
    "            )\n",
    "\n",
    "# Post processing\n",
    "best_values = tuner.get_best_hyperparameters()[0].values\n",
    "\n",
    "best_model = tuner.get_best_models(1)[0]\n",
    "res = np.concatenate(best_model.predict(datasets_final['train']), -1)\n",
    "# best_model.save(f'models/meta_model_{what}/', include_optimizer=False)\n",
    "\n",
    "\n",
    "_df = pd.DataFrame(res, columns=['pred', 'target', 'prop', 'bdid','c1', 'selector_0', 'selector_1'])\n",
    "_df['prop'] = _df.prop.apply(lambda x: x.decode('utf-8'))\n",
    "# _df['bdid'] = _df.prop.apply(lambda x: x.decode('utf-8'))\n",
    "\n",
    "props = _df.prop.unique()\n",
    "res_Tg = _df[_df['prop'] == 'Tg']\n",
    "res_Tm = _df[_df['prop'] == 'Tm']      \n",
    "\n",
    "y_true = np.array(res_Tm.target.values, dtype = int)\n",
    "y_scores = res_Tm.pred.values\n",
    "fpr, tpr, thresholds = metrics.roc_curve(y_true, y_scores, pos_label = 1)\n",
    "optimal_idx = np.argmax(tpr - fpr)\n",
    "optimal_threshold = thresholds[optimal_idx]\n",
    "predicted_value = []\n",
    "for i in res_Tm.pred.values:\n",
    "    if i > optimal_threshold:\n",
    "        predicted_value.append(1)\n",
    "    else:\n",
    "        predicted_value.append(0)\n",
    "m = tf.keras.metrics.Precision()\n",
    "m.update_state(np.array(res_Tm.target.values, dtype = int), predicted_value)\n",
    "prec_Tm.append(m.result().numpy())\n",
    "m = tf.keras.metrics.Recall()\n",
    "m.update_state(np.array(res_Tm.target.values, dtype = int), predicted_value)\n",
    "recall_Tm.append(m.result().numpy())\n",
    "m = tf.keras.metrics.Accuracy()\n",
    "m.update_state(np.array(res_Tm.target.values, dtype = int), predicted_value)\n",
    "accuracy_Tm.append(m.result().numpy())\n",
    "\n",
    "y_true = np.array(res_Tg.target.values, dtype = int)\n",
    "y_scores = res_Tg.pred.values\n",
    "fpr, tpr, thresholds = metrics.roc_curve(y_true, y_scores, pos_label = 1)\n",
    "optimal_idx = np.argmax(tpr - fpr)\n",
    "optimal_threshold = thresholds[optimal_idx]\n",
    "predicted_value = []\n",
    "for i in res_Tg.pred.values:\n",
    "    if i > optimal_threshold:\n",
    "        predicted_value.append(1)\n",
    "    else:\n",
    "        predicted_value.append(0)\n",
    "m = tf.keras.metrics.Precision()\n",
    "m.update_state(np.array(res_Tg.target.values, dtype = int), predicted_value)\n",
    "prec_Tg.append(m.result().numpy())\n",
    "m = tf.keras.metrics.Recall()\n",
    "m.update_state(np.array(res_Tg.target.values, dtype = int), predicted_value)\n",
    "recall_Tg.append(m.result().numpy())\n",
    "m = tf.keras.metrics.Accuracy()\n",
    "m.update_state(np.array(res_Tg.target.values, dtype = int), predicted_value)\n",
    "accuracy_Tg.append(m.result().numpy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_performance = pd.DataFrame({'prec_Tg' :prec_Tg, 'recall_Tg' : recall_Tg, 'accuracy_Tg' : accuracy_Tg, 'prec_Tm' :prec_Tm, 'recall_Tm' : recall_Tm, 'accuracy_Tm' : accuracy_Tm }, columns=['prec_Tg','recall_Tg', 'accuracy_Tg', 'prec_Tm','recall_Tm', 'accuracy_Tm'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_performance['f1_score_Tg'] = 2*df_performance.prec_Tg*df_performance.recall_Tg/(df_performance.prec_Tg + df_performance.recall_Tg)\n",
    "df_performance['f1_score_Tm'] = 2*df_performance.prec_Tm*df_performance.recall_Tm/(df_performance.prec_Tm + df_performance.recall_Tm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prec_Tg</th>\n",
       "      <th>recall_Tg</th>\n",
       "      <th>accuracy_Tg</th>\n",
       "      <th>prec_Tm</th>\n",
       "      <th>recall_Tm</th>\n",
       "      <th>accuracy_Tm</th>\n",
       "      <th>f1_score_Tg</th>\n",
       "      <th>f1_score_Tm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.976536</td>\n",
       "      <td>0.87751</td>\n",
       "      <td>0.878505</td>\n",
       "      <td>0.969512</td>\n",
       "      <td>0.797992</td>\n",
       "      <td>0.826794</td>\n",
       "      <td>0.924379</td>\n",
       "      <td>0.87543</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    prec_Tg  recall_Tg  accuracy_Tg   prec_Tm  recall_Tm  accuracy_Tm  \\\n",
       "0  0.976536    0.87751     0.878505  0.969512   0.797992     0.826794   \n",
       "\n",
       "   f1_score_Tg  f1_score_Tm  \n",
       "0     0.924379      0.87543  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'num_layers': 2,\n",
       " 'units_0': 512,\n",
       " 'dropout_0': 0.5,\n",
       " 'learning_rate': 0.001,\n",
       " 'units_1': 256,\n",
       " 'dropout_1': 0.1,\n",
       " 'tuner/epochs': 300,\n",
       " 'tuner/initial_epoch': 0,\n",
       " 'tuner/bracket': 0,\n",
       " 'tuner/round': 0}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_values"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a686d76ec85f590216c9e7e08a09f9fc1c5b481b97ecdbc5e95356f9feef4f2c"
  },
  "kernelspec": {
   "display_name": "Python 3.8.7 ('.venv': poetry)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  },
  "output_auto_scroll": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}