02. train_image_extracted.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 02. Image Embedding\n",
    "\n",
    "This notebook is used for generating the below files:\n",
    "\n",
    "1. `corruption_free_coco_descriptions.pkl`\n",
    "2. `train_image_extracted.pkl`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6 miniproject/sem6/wsl/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import time\n",
    "import numpy as np\n",
    "from keras.applications.inception_v3 import InceptionV3,preprocess_input\n",
    "from keras.layers import Dense,BatchNormalization,Dropout,Embedding,RepeatVector\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.models import Sequential\n",
    "from keras.models import Model\n",
    "\n",
    "from pycocotools.coco import COCO\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataDir='coco'\n",
    "dataType='train2014'\n",
    "annFile='{}/annotations/captions_{}.json'.format(dataDir,dataType)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coco=COCO(annFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since we are using this as feature extractor, the last softmax layer is not useful for us.\n",
    "inception = InceptionV3(weights='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inception.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pop the last softmax layer and freezing the remaining layers\n",
    "inception.layers.pop()\n",
    "\n",
    "for layer in inception.layers:\n",
    "    layer.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'mixed10/concat:0' shape=(None, 8, 8, 2048) dtype=float32>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inception.layers[-2].output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### image ==> [Inception + Embedding ] ==> Feature Extracted image of shape (None,300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/Kanishk/Documents/Projects/sem6/wsl/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"av...)`\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# building the final model\n",
    "final_model = Model(input = inception.input,output = inception.layers[-1].output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "TARGET_SIZE = (299,299)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getImgPath(imgId):\n",
    "    padding = \"0\" * (12  - len(str(imgId)))\n",
    "    imgName = \"{}{}\".format(padding, imgId)\n",
    "    return \"{}/{}/COCO_{}_{}.jpg\".format(dataDir, dataType, dataType, imgName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'coco/train2014/COCO_train2014_000000001036.jpg'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getImgPath(1036)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Finding corrupt images\n",
    "Certain images not extracted properly may give \"**image file is truncated (x bytes not processed)**\"\n",
    "\n",
    "Loop for finding all corrupt images and adding them to a list\n",
    "\n",
    "May take anywhere between 40-45mins (dependent on specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Finding Corrupt Images: 100%|##########| 82783/82783 [43:25<00:00, 31.78it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 82783 images in 2605.21s\n",
      "Found 1 corrupt images\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "corrupt_images = []\n",
    "start = time.time()\n",
    "imgIds = coco.getImgIds()\n",
    "\n",
    "for imgId in tqdm(imgIds, ascii=True, desc=\"Finding Corrupt Images\"):\n",
    "    try:\n",
    "        load_img(getImgPath(imgId), target_size=TARGET_SIZE)\n",
    "    except OSError:\n",
    "        corrupt_images.append(imgId)\n",
    "\n",
    "print(\"Processed {} images in {:.2f}s\".format(len(imgIds), time.time() - start))\n",
    "print(\"Found {} corrupt image(s)\".format(len(corrupt_images)))\n",
    "\n",
    "print(\"Path of corrupted files:\")\n",
    "for imgId in corrupt_images:\n",
    "    print(getImgPath(imgId))\n",
    "corrupt_images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove corrupt images' descriptions.\n",
    "\n",
    "Load `coco_descriptions.pkl` we created in `pycocoDescriptions` notebook and save a new pickle with corrupt images' captions removed into `corruption_free_coco_descriptions.pkl`.\n",
    "\n",
    "Note that you use the new file while training..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size before removing corrupt images: 82783\n",
      "Size after removing corrupt images: 82782\n",
      "Saving new file with corrupt image captions removed...\n"
     ]
    }
   ],
   "source": [
    "with open('coco_descriptions.pkl', 'rb') as f:\n",
    "    descriptions = pickle.load(f)\n",
    "\n",
    "print(\"Size before removing corrupt images: {}\".format(len(descriptions)))\n",
    "for imgId in corrupt_images:\n",
    "    descriptions.pop(imgId)\n",
    "print(\"Size after removing corrupt images: {}\".format(len(descriptions)))\n",
    "\n",
    "print(\"Saving new file with corrupt image captions removed...\")\n",
    "with open('corruption_free_coco_descriptions.pkl', 'wb') as f:\n",
    "    pickle.dump(descriptions, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing corrupt image Ids from `coco.getImgIds()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "82783\n",
      "82782\n"
     ]
    }
   ],
   "source": [
    "imgIds = coco.getImgIds()\n",
    "\n",
    "print(len(imgIds))\n",
    "for imgId in corrupt_images:\n",
    "    imgIds.remove(imgId)\n",
    "\n",
    "print(len(imgIds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating `train_image_extracted.pkl`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code for image imbedding i.e converting image to 300 dimentional\n",
    "train_image_extracted = dict()\n",
    "\n",
    "try:\n",
    "    for imgId in tqdm(imgIds, ascii=True, desc=\"Generating Input Matrix\"):\n",
    "        img = load_img(getImgPath(imgId), target_size=TARGET_SIZE)\n",
    "        # Converting image to array\n",
    "        img_array = img_to_array(img)\n",
    "        nimage = preprocess_input(img_array)\n",
    "        # Adding one more dimesion\n",
    "        nimage = np.expand_dims(nimage, axis=0)    \n",
    "        fea_vec = final_model.predict(nimage)\n",
    "        train_image_extracted[imgId] = np.reshape(fea_vec, fea_vec.shape[1]) # reshape from (1, 2048) to (2048, )\n",
    "\n",
    "\n",
    "except Exception as e:\n",
    "    print(\"Exception got :- \\n\",e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_embedding_matrix(matrix):\n",
    "    with open(\"train_image_extracted.pkl\",\"wb\") as f:\n",
    "        pickle.dump(matrix, f)\n",
    "\n",
    "save_embedding_matrix(train_image_extracted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 2048)\n"
     ]
    }
   ],
   "source": [
    "print(final_model.output.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}