06_03_qa_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "import os\n",
    "\n",
    "from tensorflow.keras.layers import Input, Embedding, GRU, GRUCell, Bidirectional, TimeDistributed, Dense, Lambda\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.utils import plot_model\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "from utils.write import training_data, test_data, collapse_documents, expand_answers, _read_data, get_glove, START_TOKEN, END_TOKEN, look_up_token\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run params\n",
    "SECTION = 'write'\n",
    "RUN_ID = '0001'\n",
    "DATA_NAME = 'qa'\n",
    "RUN_FOLDER = 'run/{}/'.format(SECTION)\n",
    "RUN_FOLDER += '_'.join([RUN_ID, DATA_NAME])\n",
    "\n",
    "if not os.path.exists(RUN_FOLDER):\n",
    "    os.mkdir(RUN_FOLDER)\n",
    "    os.mkdir(os.path.join(RUN_FOLDER, 'viz'))\n",
    "    os.mkdir(os.path.join(RUN_FOLDER, 'images'))\n",
    "    os.mkdir(os.path.join(RUN_FOLDER, 'weights'))\n",
    "\n",
    "mode =  'build' #'load' #"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### LOAD DATA ####\n",
    "\n",
    "test_data_gen = test_data()\n",
    "batch = next(test_data_gen)\n",
    "batch = collapse_documents(batch)\n",
    "glove = get_glove()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE = glove.shape[0]\n",
    "EMBEDDING_DIMENS = glove.shape[1]\n",
    "\n",
    "GRU_UNITS = 100\n",
    "MAX_DOC_SIZE = None\n",
    "MAX_ANSWER_SIZE = None\n",
    "MAX_Q_SIZE = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### TRAINING MODEL ####\n",
    "\n",
    "document_tokens = Input(shape=(MAX_DOC_SIZE,), name=\"document_tokens\")\n",
    "\n",
    "embedding = Embedding(input_dim = VOCAB_SIZE, output_dim = EMBEDDING_DIMENS, weights=[glove], mask_zero = True, name = 'embedding')\n",
    "document_emb = embedding(document_tokens)\n",
    "\n",
    "answer_outputs = Bidirectional(GRU(GRU_UNITS, return_sequences=True), name = 'answer_outputs')(document_emb)\n",
    "answer_tags = Dense(2, activation = 'softmax', name = 'answer_tags')(answer_outputs)\n",
    "\n",
    "encoder_input_mask = Input(shape=(MAX_ANSWER_SIZE, MAX_DOC_SIZE), name=\"encoder_input_mask\")\n",
    "encoder_inputs = Lambda(lambda x: K.batch_dot(x[0], x[1]), name=\"encoder_inputs\")([encoder_input_mask, answer_outputs])\n",
    "encoder_cell = GRU(2 * GRU_UNITS, name = 'encoder_cell')(encoder_inputs)\n",
    "\n",
    "decoder_inputs = Input(shape=(MAX_Q_SIZE,), name=\"decoder_inputs\")\n",
    "decoder_emb = embedding(decoder_inputs)\n",
    "decoder_emb.trainable = False\n",
    "decoder_cell = GRU(2 * GRU_UNITS, return_sequences = True, name = 'decoder_cell')\n",
    "decoder_states = decoder_cell(decoder_emb, initial_state = [encoder_cell])\n",
    "\n",
    "decoder_projection = Dense(VOCAB_SIZE, name = 'decoder_projection', activation = 'softmax', use_bias = False)\n",
    "decoder_outputs = decoder_projection(decoder_states)\n",
    "\n",
    "total_model = Model([document_tokens, decoder_inputs, encoder_input_mask], [answer_tags, decoder_outputs])\n",
    "plot_model(total_model, to_file=os.path.join(RUN_FOLDER ,'viz/model.png'),show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### INFERENCE MODEL ####\n",
    "\n",
    "decoder_inputs_dynamic = Input(shape=(1,), name=\"decoder_inputs_dynamic\")\n",
    "decoder_emb_dynamic = embedding(decoder_inputs_dynamic)\n",
    "decoder_init_state_dynamic = Input(shape=(2 * GRU_UNITS,), name = 'decoder_init_state_dynamic') #the embedding of the previous word\n",
    "decoder_states_dynamic = decoder_cell(decoder_emb_dynamic, initial_state = [decoder_init_state_dynamic])\n",
    "decoder_outputs_dynamic = decoder_projection(decoder_states_dynamic)\n",
    "\n",
    "answer_model = Model(document_tokens, [answer_tags])\n",
    "decoder_initial_state_model = Model([document_tokens, encoder_input_mask], [encoder_cell])\n",
    "question_model = Model([decoder_inputs_dynamic, decoder_init_state_dynamic], [decoder_outputs_dynamic, decoder_states_dynamic])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### LOAD MODEL WEIGHTS ####\n",
    "\n",
    "model_num = 1\n",
    "\n",
    "total_model.load_weights(os.path.join(RUN_FOLDER, 'weights/weights_{}.h5'.format(model_num)), by_name = True)\n",
    "question_model.load_weights(os.path.join(RUN_FOLDER, 'weights/weights_{}.h5'.format(model_num)), by_name = True)\n",
    "answer_model.load_weights(os.path.join(RUN_FOLDER, 'weights/weights_{}.h5'.format(model_num)), by_name = True)\n",
    "decoder_initial_state_model.load_weights(os.path.join(RUN_FOLDER, 'weights/weights_{}.h5'.format(model_num)), by_name = True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# answer placement predictions\n",
    "\n",
    "plt.figure(figsize=(15,5))\n",
    "idx = 0\n",
    "\n",
    "answer_preds = answer_model.predict(batch[\"document_tokens\"])\n",
    "\n",
    "print('Predicted answer probabilities')\n",
    "ax = plt.gca()\n",
    "ax.xaxis.grid(True)\n",
    "plt.plot(answer_preds[idx, :, 1])\n",
    "plt.show()\n",
    "\n",
    "for i in range(len(batch['document_words'][idx])):\n",
    "    print(i, batch['document_words'][idx][i], np.round(answer_preds[idx][i][1],2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set chosen answer position\n",
    "\n",
    "start_answer = 37\n",
    "end_answer = 39\n",
    "\n",
    "print(batch['document_words'][idx][start_answer:(1+end_answer)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "answer_preds = answer_model.predict(batch[\"document_tokens\"])\n",
    "\n",
    "answers = [[0] * len(answer_preds[idx])]\n",
    "for i in range(start_answer, end_answer + 1):\n",
    "    answers[0][i] = 1\n",
    "\n",
    "answer_batch = expand_answers(batch, answers)\n",
    "\n",
    "next_decoder_init_state = decoder_initial_state_model.predict([answer_batch['document_tokens'][[idx]], answer_batch['answer_masks'][[idx]]])\n",
    "\n",
    "word_tokens = [START_TOKEN]\n",
    "questions = [look_up_token(START_TOKEN)]\n",
    "\n",
    "ended = False\n",
    "counter = 0\n",
    "\n",
    "while not ended:\n",
    "    \n",
    "    counter += 1\n",
    "\n",
    "    word_preds, next_decoder_init_state = question_model.predict([np.array(word_tokens), next_decoder_init_state])\n",
    "\n",
    "    next_decoder_init_state = np.squeeze(next_decoder_init_state, axis = 1)\n",
    "    word_tokens = np.argmax(word_preds, 2)[0]\n",
    "\n",
    "    questions.append(look_up_token(word_tokens[0]))\n",
    "\n",
    "    if word_tokens[0] == END_TOKEN or counter > 20 :\n",
    "        ended = True\n",
    "\n",
    "questions = ' '.join(questions)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gdl_code",
   "language": "python",
   "name": "gdl_code"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}