training_with_keras_tuner.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "training_with_keras_tuner.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOTnlyy6XKirhwmm2W7Flff",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cs145442/nlp-projects-with-tf2/blob/master/training_with_keras_tuner.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VKa7r6VxHGd9"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "import IPython"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pb7ibfT4HJ_t",
        "outputId": "31596bc0-eae7-4399-9a9f-d029d9242043"
      },
      "source": [
        "!pip install -q -U keras-tuner\n",
        "import kerastuner as kt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |██████                          | 10kB 14.1MB/s eta 0:00:01\r\u001b[K     |████████████                    | 20kB 17.6MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 30kB 11.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 40kB 9.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 51kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 61kB 3.4MB/s \n",
            "\u001b[?25h  Building wheel for keras-tuner (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for terminaltables (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xQbNZyZjHQog",
        "outputId": "3d51e8af-9a00-4a7d-cbb9-3be4a319e7dd"
      },
      "source": [
        "(img_train, label_train), (img_test, label_test) = keras.datasets.fashion_mnist.load_data()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
            "32768/29515 [=================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
            "26427392/26421880 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
            "8192/5148 [===============================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
            "4423680/4422102 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VOvPfUciHT7H"
      },
      "source": [
        "# Normalize pixel values between 0 and 1\n",
        "img_train = img_train.astype('float32') / 255.0\n",
        "img_test = img_test.astype('float32') / 255.0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d9P-viGfHV59"
      },
      "source": [
        "def model_builder(hp):\n",
        "  model = keras.Sequential()\n",
        "  model.add(keras.layers.Flatten(input_shape=(28, 28)))\n",
        "\n",
        "  # Tune the number of units in the first Dense layer\n",
        "  # Choose an optimal value between 32-512\n",
        "  hp_units = hp.Int('units', min_value = 32, max_value = 512, step = 32)\n",
        "  model.add(keras.layers.Dense(units = hp_units, activation = 'relu'))\n",
        "  model.add(keras.layers.Dense(10))\n",
        "\n",
        "  # Tune the learning rate for the optimizer \n",
        "  # Choose an optimal value from 0.01, 0.001, or 0.0001\n",
        "  hp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-3, 1e-4]) \n",
        "\n",
        "  model.compile(optimizer = keras.optimizers.Adam(learning_rate = hp_learning_rate),\n",
        "                loss = keras.losses.SparseCategoricalCrossentropy(from_logits = True), \n",
        "                metrics = ['accuracy'])\n",
        "\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J6alNoleHjHm"
      },
      "source": [
        "tuner = kt.Hyperband(model_builder,\n",
        "                     objective = 'val_accuracy', \n",
        "                     max_epochs = 10,\n",
        "                     factor = 3,\n",
        "                     directory = 'my_dir',\n",
        "                     project_name = 'intro_to_kt')                       "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CIq3EUn6H0aq"
      },
      "source": [
        "class ClearTrainingOutput(tf.keras.callbacks.Callback):\n",
        "  def on_train_end(*args, **kwargs):\n",
        "    IPython.display.clear_output(wait = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "UDlLtWEAIQNR",
        "outputId": "7aafed24-4216-49a4-9f31-64f2f60f0061"
      },
      "source": [
        "tuner.search(img_train, label_train, epochs = 10, validation_data = (img_test, label_test), callbacks = [ClearTrainingOutput()])\n",
        "\n",
        "# Get the optimal hyperparameters\n",
        "best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]\n",
        "\n",
        "print(f\"\"\"\n",
        "The hyperparameter search is complete. The optimal number of units in the first densely-connected\n",
        "layer is {best_hps.get('units')} and the optimal learning rate for the optimizer\n",
        "is {best_hps.get('learning_rate')}.\n",
        "\"\"\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:#4527A0\"><h1 style=\"font-size:18px\">Trial complete</h1></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:#4527A0\"><h1 style=\"font-size:18px\">Trial summary</h1></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-Trial ID: f4c42803f24e7b25e82398d00e59d641</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-Score: 0.8863000273704529</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-Best step: 0</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:#7E57C2\"><h2 style=\"font-size:16px\">Hyperparameters:</h2></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-learning_rate: 0.001</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-tuner/bracket: 0</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-tuner/epochs: 10</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-tuner/initial_epoch: 0</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-tuner/round: 0</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-units: 384</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Oracle triggered exit\n",
            "\n",
            "The hyperparameter search is complete. The optimal number of units in the first densely-connected\n",
            "layer is 384 and the optimal learning rate for the optimizer\n",
            "is 0.001.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CrQHs2-3Ih4l",
        "outputId": "fafdb268-1770-4670-9077-76599471a174"
      },
      "source": [
        "# Build the model with the optimal hyperparameters and train it on the data\n",
        "model = tuner.hypermodel.build(best_hps)\n",
        "model.fit(img_train, label_train, epochs = 10, validation_data = (img_test, label_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.4747 - accuracy: 0.8304 - val_loss: 0.4310 - val_accuracy: 0.8481\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.3606 - accuracy: 0.8690 - val_loss: 0.3635 - val_accuracy: 0.8680\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.3201 - accuracy: 0.8819 - val_loss: 0.3911 - val_accuracy: 0.8541\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.2978 - accuracy: 0.8894 - val_loss: 0.3352 - val_accuracy: 0.8786\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.2802 - accuracy: 0.8972 - val_loss: 0.3427 - val_accuracy: 0.8770\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.2645 - accuracy: 0.9012 - val_loss: 0.3270 - val_accuracy: 0.8843\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2522 - accuracy: 0.9063 - val_loss: 0.3459 - val_accuracy: 0.8796\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.2423 - accuracy: 0.9093 - val_loss: 0.3408 - val_accuracy: 0.8830\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2300 - accuracy: 0.9134 - val_loss: 0.3271 - val_accuracy: 0.8888\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 6s 3ms/step - loss: 0.2211 - accuracy: 0.9166 - val_loss: 0.3392 - val_accuracy: 0.8820\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7ff167336f60>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iouv9m5KIpl6"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}