keras-bert.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import tensorflow_hub as hub\n",
        "import os\n",
        "import re\n",
        "import numpy as np\n",
        "from bert.tokenization import FullTokenizer\n",
        "from tqdm import tqdm_notebook\n",
        "from tensorflow.keras import backend as K\n",
        "\n",
        "# Initialize session\n",
        "sess = tf.Session()\n",
        "\n",
        "# Params for bert model and tokenization\n",
        "bert_path = \"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\"\n",
        "max_seq_length = 256"
      ],
      "outputs": [],
      "execution_count": 1,
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data\n",
        "\nFirst, we load the sample data IMDB data"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "# Load all files from a directory in a DataFrame.\n",
        "def load_directory_data(directory):\n",
        "  data = {}\n",
        "  data[\"sentence\"] = []\n",
        "  data[\"sentiment\"] = []\n",
        "  for file_path in os.listdir(directory):\n",
        "    with tf.gfile.GFile(os.path.join(directory, file_path), \"r\") as f:\n",
        "      data[\"sentence\"].append(f.read())\n",
        "      data[\"sentiment\"].append(re.match(\"\\d+_(\\d+)\\.txt\", file_path).group(1))\n",
        "  return pd.DataFrame.from_dict(data)\n",
        "\n",
        "# Merge positive and negative examples, add a polarity column and shuffle.\n",
        "def load_dataset(directory):\n",
        "  pos_df = load_directory_data(os.path.join(directory, \"pos\"))\n",
        "  neg_df = load_directory_data(os.path.join(directory, \"neg\"))\n",
        "  pos_df[\"polarity\"] = 1\n",
        "  neg_df[\"polarity\"] = 0\n",
        "  return pd.concat([pos_df, neg_df]).sample(frac=1).reset_index(drop=True)\n",
        "\n",
        "# Download and process the dataset files.\n",
        "def download_and_load_datasets(force_download=False):\n",
        "  dataset = tf.keras.utils.get_file(\n",
        "      fname=\"aclImdb.tar.gz\", \n",
        "      origin=\"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\", \n",
        "      extract=True)\n",
        "\n",
        "  train_df = load_dataset(os.path.join(os.path.dirname(dataset), \n",
        "                                       \"aclImdb\", \"train\"))\n",
        "  test_df = load_dataset(os.path.join(os.path.dirname(dataset), \n",
        "                                      \"aclImdb\", \"test\"))\n",
        "\n",
        "  return train_df, test_df\n",
        "\n",
        "# Reduce logging output.\n",
        "tf.logging.set_verbosity(tf.logging.ERROR)\n",
        "\n",
        "train_df, test_df = download_and_load_datasets()\n",
        "train_df.head()"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 2,
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>polarity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This is a truly remarkable piece of cinematic ...</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>This movie wasn't that bad when compared to th...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>A delightful gentle comedic gem, until the las...</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I watched this last night after not having see...</td>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>This movie has it all. Sight gags, subtle joke...</td>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            sentence sentiment  polarity\n",
              "0  This is a truly remarkable piece of cinematic ...        10         1\n",
              "1  This movie wasn't that bad when compared to th...         3         0\n",
              "2  A delightful gentle comedic gem, until the las...         7         1\n",
              "3  I watched this last night after not having see...         8         1\n",
              "4  This movie has it all. Sight gags, subtle joke...         8         1"
            ]
          },
          "metadata": {}
        }
      ],
      "execution_count": 2,
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "# Create datasets (Only take up to max_seq_length words for memory)\n",
        "train_text = train_df['sentence'].tolist()\n",
        "train_text = [' '.join(t.split()[0:max_seq_length]) for t in train_text]\n",
        "train_text = np.array(train_text, dtype=object)[:, np.newaxis]\n",
        "train_label = train_df['polarity'].tolist()\n",
        "\n",
        "test_text = test_df['sentence'].tolist()\n",
        "test_text = [' '.join(t.split()[0:max_seq_length]) for t in test_text]\n",
        "test_text = np.array(test_text, dtype=object)[:, np.newaxis]\n",
        "test_label = test_df['polarity'].tolist()"
      ],
      "outputs": [],
      "execution_count": 3,
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenize\n",
        "\nNext, tokenize our text to create `input_ids`, `input_masks`, and `segment_ids`"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "class PaddingInputExample(object):\n",
        "    \"\"\"Fake example so the num input examples is a multiple of the batch size.\n",
        "  When running eval/predict on the TPU, we need to pad the number of examples\n",
        "  to be a multiple of the batch size, because the TPU requires a fixed batch\n",
        "  size. The alternative is to drop the last batch, which is bad because it means\n",
        "  the entire output data won't be generated.\n",
        "  We use this class instead of `None` because treating `None` as padding\n",
        "  battches could cause silent errors.\n",
        "  \"\"\"\n",
        "\n",
        "class InputExample(object):\n",
        "    \"\"\"A single training/test example for simple sequence classification.\"\"\"\n",
        "\n",
        "    def __init__(self, guid, text_a, text_b=None, label=None):\n",
        "        \"\"\"Constructs a InputExample.\n",
        "    Args:\n",
        "      guid: Unique id for the example.\n",
        "      text_a: string. The untokenized text of the first sequence. For single\n",
        "        sequence tasks, only this sequence must be specified.\n",
        "      text_b: (Optional) string. The untokenized text of the second sequence.\n",
        "        Only must be specified for sequence pair tasks.\n",
        "      label: (Optional) string. The label of the example. This should be\n",
        "        specified for train and dev examples, but not for test examples.\n",
        "    \"\"\"\n",
        "        self.guid = guid\n",
        "        self.text_a = text_a\n",
        "        self.text_b = text_b\n",
        "        self.label = label\n",
        "\n",
        "def create_tokenizer_from_hub_module():\n",
        "    \"\"\"Get the vocab file and casing info from the Hub module.\"\"\"\n",
        "    bert_module =  hub.Module(bert_path)\n",
        "    tokenization_info = bert_module(signature=\"tokenization_info\", as_dict=True)\n",
        "    vocab_file, do_lower_case = sess.run(\n",
        "        [\n",
        "            tokenization_info[\"vocab_file\"],\n",
        "            tokenization_info[\"do_lower_case\"],\n",
        "        ]\n",
        "    )\n",
        "\n",
        "    return FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)\n",
        "\n",
        "def convert_single_example(tokenizer, example, max_seq_length=256):\n",
        "    \"\"\"Converts a single `InputExample` into a single `InputFeatures`.\"\"\"\n",
        "\n",
        "    if isinstance(example, PaddingInputExample):\n",
        "        input_ids = [0] * max_seq_length\n",
        "        input_mask = [0] * max_seq_length\n",
        "        segment_ids = [0] * max_seq_length\n",
        "        label = 0\n",
        "        return input_ids, input_mask, segment_ids, label\n",
        "\n",
        "    tokens_a = tokenizer.tokenize(example.text_a)\n",
        "    if len(tokens_a) > max_seq_length - 2:\n",
        "        tokens_a = tokens_a[0 : (max_seq_length - 2)]\n",
        "\n",
        "    tokens = []\n",
        "    segment_ids = []\n",
        "    tokens.append(\"[CLS]\")\n",
        "    segment_ids.append(0)\n",
        "    for token in tokens_a:\n",
        "        tokens.append(token)\n",
        "        segment_ids.append(0)\n",
        "    tokens.append(\"[SEP]\")\n",
        "    segment_ids.append(0)\n",
        "\n",
        "    input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
        "    # tokens are attended to.\n",
        "    input_mask = [1] * len(input_ids)\n",
        "\n",
        "    # Zero-pad up to the sequence length.\n",
        "    while len(input_ids) < max_seq_length:\n",
        "        input_ids.append(0)\n",
        "        input_mask.append(0)\n",
        "        segment_ids.append(0)\n",
        "\n",
        "    assert len(input_ids) == max_seq_length\n",
        "    assert len(input_mask) == max_seq_length\n",
        "    assert len(segment_ids) == max_seq_length\n",
        "\n",
        "    return input_ids, input_mask, segment_ids, example.label\n",
        "\n",
        "def convert_examples_to_features(tokenizer, examples, max_seq_length=256):\n",
        "    \"\"\"Convert a set of `InputExample`s to a list of `InputFeatures`.\"\"\"\n",
        "\n",
        "    input_ids, input_masks, segment_ids, labels = [], [], [], []\n",
        "    for example in tqdm_notebook(examples, desc=\"Converting examples to features\"):\n",
        "        input_id, input_mask, segment_id, label = convert_single_example(\n",
        "            tokenizer, example, max_seq_length\n",
        "        )\n",
        "        input_ids.append(input_id)\n",
        "        input_masks.append(input_mask)\n",
        "        segment_ids.append(segment_id)\n",
        "        labels.append(label)\n",
        "    return (\n",
        "        np.array(input_ids),\n",
        "        np.array(input_masks),\n",
        "        np.array(segment_ids),\n",
        "        np.array(labels).reshape(-1, 1),\n",
        "    )\n",
        "\n",
        "def convert_text_to_examples(texts, labels):\n",
        "    \"\"\"Create InputExamples\"\"\"\n",
        "    InputExamples = []\n",
        "    for text, label in zip(texts, labels):\n",
        "        InputExamples.append(\n",
        "            InputExample(guid=None, text_a=\" \".join(text), text_b=None, label=label)\n",
        "        )\n",
        "    return InputExamples\n",
        "\n",
        "# Instantiate tokenizer\n",
        "tokenizer = create_tokenizer_from_hub_module()\n",
        "\n",
        "# Convert data to InputExample format\n",
        "train_examples = convert_text_to_examples(train_text, train_label)\n",
        "test_examples = convert_text_to_examples(test_text, test_label)\n",
        "\n",
        "# Convert to features\n",
        "(train_input_ids, train_input_masks, train_segment_ids, train_labels \n",
        ") = convert_examples_to_features(tokenizer, train_examples, max_seq_length=max_seq_length)\n",
        "(test_input_ids, test_input_masks, test_segment_ids, test_labels\n",
        ") = convert_examples_to_features(tokenizer, test_examples, max_seq_length=max_seq_length)"
      ],
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "74b8910e57444f11b1a1dd431b9e8db9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Converting examples to features', max=25000, style=ProgressSt…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6ba8453dcfd74cd6a984e95b9173b1da",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Converting examples to features', max=25000, style=ProgressSt…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        }
      ],
      "execution_count": 4,
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "class BertLayer(tf.keras.layers.Layer):\n",
        "    def __init__(\n",
        "        self,\n",
        "        n_fine_tune_layers=10,\n",
        "        pooling=\"first\",\n",
        "        bert_path=\"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\",\n",
        "        **kwargs,\n",
        "    ):\n",
        "        self.n_fine_tune_layers = n_fine_tune_layers\n",
        "        self.trainable = True\n",
        "        self.output_size = 768\n",
        "        self.pooling = pooling\n",
        "        self.bert_path = bert_path\n",
        "        if self.pooling not in [\"first\", \"mean\"]:\n",
        "            raise NameError(\n",
        "                f\"Undefined pooling type (must be either first or mean, but is {self.pooling}\"\n",
        "            )\n",
        "\n",
        "        super(BertLayer, self).__init__(**kwargs)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        self.bert = hub.Module(\n",
        "            self.bert_path, trainable=self.trainable, name=f\"{self.name}_module\"\n",
        "        )\n",
        "\n",
        "        # Remove unused layers\n",
        "        trainable_vars = self.bert.variables\n",
        "        if self.pooling == \"first\":\n",
        "            trainable_vars = [var for var in trainable_vars if not \"/cls/\" in var.name]\n",
        "            trainable_layers = [\"pooler/dense\"]\n",
        "\n",
        "        elif self.pooling == \"mean\":\n",
        "            trainable_vars = [\n",
        "                var\n",
        "                for var in trainable_vars\n",
        "                if not \"/cls/\" in var.name and not \"/pooler/\" in var.name\n",
        "            ]\n",
        "            trainable_layers = []\n",
        "        else:\n",
        "            raise NameError(\n",
        "                f\"Undefined pooling type (must be either first or mean, but is {self.pooling}\"\n",
        "            )\n",
        "\n",
        "        # Select how many layers to fine tune\n",
        "        for i in range(self.n_fine_tune_layers):\n",
        "            trainable_layers.append(f\"encoder/layer_{str(11 - i)}\")\n",
        "\n",
        "        # Update trainable vars to contain only the specified layers\n",
        "        trainable_vars = [\n",
        "            var\n",
        "            for var in trainable_vars\n",
        "            if any([l in var.name for l in trainable_layers])\n",
        "        ]\n",
        "\n",
        "        # Add to trainable weights\n",
        "        for var in trainable_vars:\n",
        "            self._trainable_weights.append(var)\n",
        "\n",
        "        for var in self.bert.variables:\n",
        "            if var not in self._trainable_weights:\n",
        "                self._non_trainable_weights.append(var)\n",
        "\n",
        "        super(BertLayer, self).build(input_shape)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        inputs = [K.cast(x, dtype=\"int32\") for x in inputs]\n",
        "        input_ids, input_mask, segment_ids = inputs\n",
        "        bert_inputs = dict(\n",
        "            input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids\n",
        "        )\n",
        "        if self.pooling == \"first\":\n",
        "            pooled = self.bert(inputs=bert_inputs, signature=\"tokens\", as_dict=True)[\n",
        "                \"pooled_output\"\n",
        "            ]\n",
        "        elif self.pooling == \"mean\":\n",
        "            result = self.bert(inputs=bert_inputs, signature=\"tokens\", as_dict=True)[\n",
        "                \"sequence_output\"\n",
        "            ]\n",
        "\n",
        "            mul_mask = lambda x, m: x * tf.expand_dims(m, axis=-1)\n",
        "            masked_reduce_mean = lambda x, m: tf.reduce_sum(mul_mask(x, m), axis=1) / (\n",
        "                    tf.reduce_sum(m, axis=1, keepdims=True) + 1e-10)\n",
        "            input_mask = tf.cast(input_mask, tf.float32)\n",
        "            pooled = masked_reduce_mean(result, input_mask)\n",
        "        else:\n",
        "            raise NameError(f\"Undefined pooling type (must be either first or mean, but is {self.pooling}\")\n",
        "\n",
        "        return pooled\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        return (input_shape[0], self.output_size)"
      ],
      "outputs": [],
      "execution_count": 9,
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "# Build model\n",
        "def build_model(max_seq_length): \n",
        "    in_id = tf.keras.layers.Input(shape=(max_seq_length,), name=\"input_ids\")\n",
        "    in_mask = tf.keras.layers.Input(shape=(max_seq_length,), name=\"input_masks\")\n",
        "    in_segment = tf.keras.layers.Input(shape=(max_seq_length,), name=\"segment_ids\")\n",
        "    bert_inputs = [in_id, in_mask, in_segment]\n",
        "    \n",
        "    bert_output = BertLayer(n_fine_tune_layers=3, pooling=\"first\")(bert_inputs)\n",
        "    dense = tf.keras.layers.Dense(256, activation='relu')(bert_output)\n",
        "    pred = tf.keras.layers.Dense(1, activation='sigmoid')(dense)\n",
        "    \n",
        "    model = tf.keras.models.Model(inputs=bert_inputs, outputs=pred)\n",
        "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "    model.summary()\n",
        "    \n",
        "    return model\n",
        "\n",
        "def initialize_vars(sess):\n",
        "    sess.run(tf.local_variables_initializer())\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    sess.run(tf.tables_initializer())\n",
        "    K.set_session(sess)\n"
      ],
      "outputs": [],
      "execution_count": 10,
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model(max_seq_length)\n",
        "\n",
        "# Instantiate variables\n",
        "initialize_vars(sess)\n",
        "\n",
        "model.fit(\n",
        "    [train_input_ids, train_input_masks, train_segment_ids], \n",
        "    train_labels,\n",
        "    validation_data=([test_input_ids, test_input_masks, test_segment_ids], test_labels),\n",
        "    epochs=1,\n",
        "    batch_size=32\n",
        ")"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_masks (InputLayer)        [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "segment_ids (InputLayer)        [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bert_layer_7 (BertLayer)        (None, 768)          110104890   input_ids[0][0]                  \n",
            "                                                                 input_masks[0][0]                \n",
            "                                                                 segment_ids[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_12 (Dense)                (None, 256)          196864      bert_layer_7[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "dense_13 (Dense)                (None, 1)            257         dense_12[0][0]                   \n",
            "==================================================================================================\n",
            "Total params: 110,302,011\n",
            "Trainable params: 3,147,009\n",
            "Non-trainable params: 107,155,002\n",
            "__________________________________________________________________________________________________\n",
            "Train on 25000 samples, validate on 25000 samples\n",
            "24992/25000 [============================>.] - ETA: 4s - loss: 0.3306 - acc: 0.8556 "
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-827856e3678d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_input_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_input_masks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_segment_ids\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m )\n",
            "\u001b[0;32m~/anaconda3/envs/keras-bert/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    871\u001b[0m           \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    872\u001b[0m           \u001b[0mvalidation_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 873\u001b[0;31m           steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m    874\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    875\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m~/anaconda3/envs/keras-bert/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    396\u001b[0m           \u001b[0mvalidation_in_fit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    397\u001b[0m           \u001b[0mprepared_feed_values_from_dataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 398\u001b[0;31m           steps_name='validation_steps')\n\u001b[0m\u001b[1;32m    399\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_results\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    400\u001b[0m         \u001b[0mval_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mval_results\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/envs/keras-bert/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    353\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/envs/keras-bert/lib/python3.6/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3117\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3118\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3119\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3120\u001b[0m     return nest.pack_sequence_as(self._outputs_structure,\n",
            "\u001b[0;32m~/anaconda3/envs/keras-bert/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1440\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1441\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1442\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1443\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1444\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "execution_count": 14,
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "model.save('BertModel.h5')\n",
        "pre_save_preds = model.predict([test_input_ids[0:100], \n",
        "                                test_input_masks[0:100], \n",
        "                                test_segment_ids[0:100]]\n",
        "                              ) # predictions before we clear and reload model\n",
        "\n",
        "# Clear and load model\n",
        "model = None\n",
        "model = build_model(max_seq_length)\n",
        "initialize_vars(sess)\n",
        "model.load_weights('BertModel.h5')\n",
        "\n",
        "post_save_preds = model.predict([test_input_ids[0:100], \n",
        "                                test_input_masks[0:100], \n",
        "                                test_segment_ids[0:100]]\n",
        "                              ) # predictions after we clear and reload model\n",
        "all(pre_save_preds == post_save_preds) # Are they the same?"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_4\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_masks (InputLayer)        [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "segment_ids (InputLayer)        [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bert_layer_5 (BertLayer)        (None, 768)          110104890   input_ids[0][0]                  \n",
            "                                                                 input_masks[0][0]                \n",
            "                                                                 segment_ids[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_8 (Dense)                 (None, 256)          196864      bert_layer_5[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "dense_9 (Dense)                 (None, 1)            257         dense_8[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 110,302,011\n",
            "Trainable params: 3,147,009\n",
            "Non-trainable params: 107,155,002\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"model_5\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_masks (InputLayer)        [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "segment_ids (InputLayer)        [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bert_layer_6 (BertLayer)        (None, 768)          110104890   input_ids[0][0]                  \n",
            "                                                                 input_masks[0][0]                \n",
            "                                                                 segment_ids[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_10 (Dense)                (None, 256)          196864      bert_layer_6[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "dense_11 (Dense)                (None, 1)            257         dense_10[0][0]                   \n",
            "==================================================================================================\n",
            "Total params: 110,302,011\n",
            "Trainable params: 3,147,009\n",
            "Non-trainable params: 107,155,002\n",
            "__________________________________________________________________________________________________\n"
          ]
        },
        {
          "output_type": "execute_result",
          "execution_count": 13,
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {}
        }
      ],
      "execution_count": 13,
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [],
      "outputs": [],
      "execution_count": null,
      "metadata": {}
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    },
    "nteract": {
      "version": "0.12.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}