Architecture.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_nb_params_shape(shape):\n",
    "    nb_params = 1\n",
    "    for dim in shape:\n",
    "        nb_params = nb_params * int(dim)\n",
    "    return nb_params\n",
    "\n",
    "def count_number_trainable_params():\n",
    "    tot_nb_params = 0\n",
    "    for trainable_variable in slim.get_trainable_variables():\n",
    "        print(trainable_variable.name, trainable_variable.shape)\n",
    "        shape = trainable_variable.get_shape()  # e.g [D,F] or [W,H,C]\n",
    "        current_nb_params = get_nb_params_shape(shape)\n",
    "        tot_nb_params = tot_nb_params + current_nb_params\n",
    "    print('Total number of trainable params: ', tot_nb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "is_training = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Текущая модель"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convolution/Conv1/weights:0 (1, 1, 3, 32)\n",
      "Convolution/Conv1/BatchNorm/beta:0 (32,)\n",
      "Convolution/Conv2/weights:0 (3, 3, 32, 32)\n",
      "Convolution/Conv2/biases:0 (32,)\n",
      "Convolution/Conv3/weights:0 (3, 3, 32, 32)\n",
      "Convolution/Conv3/biases:0 (32,)\n",
      "GRU_RNN_cell/rnn/gru_cell/gates/weights:0 (144868, 200)\n",
      "GRU_RNN_cell/rnn/gru_cell/gates/biases:0 (200,)\n",
      "GRU_RNN_cell/rnn/gru_cell/candidate/weights:0 (144868, 100)\n",
      "GRU_RNN_cell/rnn/gru_cell/candidate/biases:0 (100,)\n",
      "Fully-connected/weights:0 (100, 6)\n",
      "Fully-connected/biases:0 (6,)\n",
      "Total number of trainable params:  43479930\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "with tf.Graph().as_default():\n",
    "    x = tf.placeholder(dtype=tf.float32, shape=(50, 240, 320, 3), name=\"input\")\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=(1,), name='labels')\n",
    "\n",
    "    with slim.arg_scope([slim.conv2d], stride=1, weights_initializer=tf.contrib.layers.xavier_initializer_conv2d()):\n",
    "        with tf.variable_scope('Convolution', [input]):\n",
    "            conv1 = slim.conv2d(x, 32, [1, 1], stride=2, scope='Conv1', normalizer_fn=slim.batch_norm,\n",
    "                                normalizer_params={'is_training': is_training})\n",
    "            pool1 = slim.max_pool2d(conv1, [3, 3], scope='Pool1', stride=1)\n",
    "            conv2 = slim.conv2d(pool1, 32, [3, 3], scope='Conv2')\n",
    "            pool2 = slim.max_pool2d(conv2, [3, 3], scope='Pool2', stride=1)\n",
    "            conv3 = slim.conv2d(pool2, 32, [3, 3], stride=2, scope='Conv3')\n",
    "    size = np.prod(conv3.get_shape().as_list()[1:])\n",
    "\n",
    "    with tf.variable_scope('GRU_RNN_cell'):\n",
    "        rnn_inputs = tf.reshape(conv3, (-1, 50, size))\n",
    "        cell = tf.contrib.rnn.GRUCell(100)\n",
    "        init_state = cell.zero_state(1, dtype=tf.float32)\n",
    "        rnn_outputs, _ = tf.nn.dynamic_rnn(cell, rnn_inputs, initial_state=init_state)\n",
    "        output = tf.reduce_mean(rnn_outputs, axis=1)\n",
    "\n",
    "    with tf.name_scope('Dense'):\n",
    "        logits = slim.fully_connected(output, 6, scope=\"Fully-connected\")\n",
    "    count_number_trainable_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Модель Inception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv2d_1a_3x3/weights:0 (3, 3, 3, 32)\n",
      "Conv2d_1a_3x3/biases:0 (32,)\n",
      "Conv2d_2a_3x3/weights:0 (3, 3, 32, 32)\n",
      "Conv2d_2a_3x3/biases:0 (32,)\n",
      "Conv2d_2b_3x3/weights:0 (3, 3, 32, 64)\n",
      "Conv2d_2b_3x3/biases:0 (64,)\n",
      "BlockInceptionA/IBranch_0/IConv2d_0a_1x1/weights:0 (1, 1, 64, 96)\n",
      "BlockInceptionA/IBranch_0/IConv2d_0a_1x1/biases:0 (96,)\n",
      "BlockInceptionA/IBranch_1/IConv2d_0a_1x1/weights:0 (1, 1, 64, 64)\n",
      "BlockInceptionA/IBranch_1/IConv2d_0a_1x1/biases:0 (64,)\n",
      "BlockInceptionA/IBranch_1/IConv2d_0b_3x3/weights:0 (3, 3, 64, 96)\n",
      "BlockInceptionA/IBranch_1/IConv2d_0b_3x3/biases:0 (96,)\n",
      "BlockInceptionA/IBranch_2/IConv2d_0a_1x1/weights:0 (1, 1, 64, 64)\n",
      "BlockInceptionA/IBranch_2/IConv2d_0a_1x1/biases:0 (64,)\n",
      "BlockInceptionA/IBranch_2/IConv2d_0b_3x3/weights:0 (3, 3, 64, 96)\n",
      "BlockInceptionA/IBranch_2/IConv2d_0b_3x3/biases:0 (96,)\n",
      "BlockInceptionA/IBranch_2/IConv2d_0c_3x3/weights:0 (3, 3, 96, 96)\n",
      "BlockInceptionA/IBranch_2/IConv2d_0c_3x3/biases:0 (96,)\n",
      "BlockInceptionA/IBranch_3/IConv2d_0b_1x1/weights:0 (1, 1, 64, 96)\n",
      "BlockInceptionA/IBranch_3/IConv2d_0b_1x1/biases:0 (96,)\n",
      "BlockReductionA/RBranch_0/RConv2d_1a_3x3/weights:0 (3, 3, 384, 384)\n",
      "BlockReductionA/RBranch_0/RConv2d_1a_3x3/biases:0 (384,)\n",
      "BlockReductionA/RBranch_1/RConv2d_0a_1x1/weights:0 (1, 1, 384, 192)\n",
      "BlockReductionA/RBranch_1/RConv2d_0a_1x1/biases:0 (192,)\n",
      "BlockReductionA/RBranch_1/RConv2d_0b_3x3/weights:0 (3, 3, 192, 224)\n",
      "BlockReductionA/RBranch_1/RConv2d_0b_3x3/biases:0 (224,)\n",
      "BlockReductionA/RBranch_1/RConv2d_1a_3x3/weights:0 (3, 3, 224, 256)\n",
      "BlockReductionA/RBranch_1/RConv2d_1a_3x3/biases:0 (256,)\n",
      "GRU_RNN_cell/rnn/gru_cell/gates/weights:0 (4632676, 200)\n",
      "GRU_RNN_cell/rnn/gru_cell/gates/biases:0 (200,)\n",
      "GRU_RNN_cell/rnn/gru_cell/candidate/weights:0 (4632676, 100)\n",
      "GRU_RNN_cell/rnn/gru_cell/candidate/biases:0 (100,)\n",
      "Fully-connected/weights:0 (100, 6)\n",
      "Fully-connected/biases:0 (6,)\n",
      "Total number of trainable params:  1392352026\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "with tf.Graph().as_default():\n",
    "\n",
    "    inputs = tf.placeholder(dtype=tf.float32, shape=(50, 240, 320, 3), name=\"input\")\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=(1,), name='labels')\n",
    "\n",
    "    # inputs = slim.conv2d(x, 32, [3, 3], scope='Conv2d_0a_1x1', stride=2, padding='VALID')\n",
    "\n",
    "    conv1 = slim.conv2d(inputs, 32, [3, 3], stride=2, padding='VALID', scope='Conv2d_1a_3x3')\n",
    "    conv2 = slim.conv2d(conv1, 32, [3, 3], padding='VALID', scope='Conv2d_2a_3x3')\n",
    "    inc_inputs = slim.conv2d(conv2, 64, [3, 3], scope='Conv2d_2b_3x3')\n",
    "\n",
    "    with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], stride=1, padding='SAME'):\n",
    "        with tf.variable_scope('BlockInceptionA', [inc_inputs]):\n",
    "            with tf.variable_scope('IBranch_0'):\n",
    "                ibranch_0 = slim.conv2d(inc_inputs, 96, [1, 1], scope='IConv2d_0a_1x1')\n",
    "            with tf.variable_scope('IBranch_1'):\n",
    "                ibranch_1_conv1 = slim.conv2d(inc_inputs, 64, [1, 1], scope='IConv2d_0a_1x1')\n",
    "                ibranch_1 = slim.conv2d(ibranch_1_conv1, 96, [3, 3], scope='IConv2d_0b_3x3')\n",
    "            with tf.variable_scope('IBranch_2'):\n",
    "                ibranch_2_conv1 = slim.conv2d(inc_inputs, 64, [1, 1], scope='IConv2d_0a_1x1')\n",
    "                ibranch_2_conv2 = slim.conv2d(ibranch_2_conv1, 96, [3, 3], scope='IConv2d_0b_3x3')\n",
    "                ibranch_2 = slim.conv2d(ibranch_2_conv2, 96, [3, 3], scope='IConv2d_0c_3x3')\n",
    "            with tf.variable_scope('IBranch_3'):\n",
    "                ibranch_3_pool = slim.avg_pool2d(inc_inputs, [3, 3], scope='IAvgPool_0a_3x3')\n",
    "                ibranch_3 = slim.conv2d(ibranch_3_pool, 96, [1, 1], scope='IConv2d_0b_1x1')\n",
    "            inception = tf.concat(axis=3, values=[ibranch_0, ibranch_1, ibranch_2, ibranch_3])\n",
    "\n",
    "    with tf.variable_scope('BlockReductionA', [inception]):\n",
    "        with tf.variable_scope('RBranch_0'):\n",
    "            rbranch_0 = slim.conv2d(inception, 384, [3, 3], stride=2, padding='VALID', scope='RConv2d_1a_3x3')\n",
    "        with tf.variable_scope('RBranch_1'):\n",
    "            rbranch_1_conv1 = slim.conv2d(inception, 192, [1, 1], scope='RConv2d_0a_1x1')\n",
    "            rbranch_1_conv2 = slim.conv2d(rbranch_1_conv1, 224, [3, 3], scope='RConv2d_0b_3x3')\n",
    "            rbranch_1 = slim.conv2d(rbranch_1_conv2, 256, [3, 3], stride=2, padding='VALID', scope='RConv2d_1a_3x3')\n",
    "        with tf.variable_scope('RBranch_2'):\n",
    "            rbranch_2 = slim.max_pool2d(inception, [3, 3], stride=2, padding='VALID', scope='RMaxPool_1a_3x3')\n",
    "        reduction = tf.concat(axis=3, values=[rbranch_0, rbranch_1, rbranch_2])\n",
    "\n",
    "    size = np.prod(reduction.get_shape().as_list()[1:])\n",
    "\n",
    "    with tf.variable_scope('GRU_RNN_cell'):\n",
    "        rnn_inputs = tf.reshape(reduction, (-1, 50, size))\n",
    "        cell = tf.contrib.rnn.GRUCell(100)\n",
    "        init_state = cell.zero_state(1, dtype=tf.float32)\n",
    "        rnn_outputs, _ = tf.nn.dynamic_rnn(cell, rnn_inputs, initial_state=init_state)\n",
    "        output = tf.reduce_mean(rnn_outputs, axis=1)\n",
    "\n",
    "    with tf.name_scope('Dense'):\n",
    "        logits = slim.fully_connected(output, 6, scope=\"Fully-connected\")\n",
    "    count_number_trainable_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VGG16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1/conv1_1/weights:0 (3, 3, 3, 64)\n",
      "conv1/conv1_1/biases:0 (64,)\n",
      "conv1/conv1_2/weights:0 (3, 3, 64, 64)\n",
      "conv1/conv1_2/biases:0 (64,)\n",
      "conv2/conv2_1/weights:0 (3, 3, 64, 128)\n",
      "conv2/conv2_1/biases:0 (128,)\n",
      "conv2/conv2_2/weights:0 (3, 3, 128, 128)\n",
      "conv2/conv2_2/biases:0 (128,)\n",
      "conv3/conv3_1/weights:0 (3, 3, 128, 256)\n",
      "conv3/conv3_1/biases:0 (256,)\n",
      "conv3/conv3_2/weights:0 (3, 3, 256, 256)\n",
      "conv3/conv3_2/biases:0 (256,)\n",
      "conv3/conv3_3/weights:0 (3, 3, 256, 256)\n",
      "conv3/conv3_3/biases:0 (256,)\n",
      "conv4/conv4_1/weights:0 (3, 3, 256, 512)\n",
      "conv4/conv4_1/biases:0 (512,)\n",
      "conv4/conv4_2/weights:0 (3, 3, 512, 512)\n",
      "conv4/conv4_2/biases:0 (512,)\n",
      "conv4/conv4_3/weights:0 (3, 3, 512, 512)\n",
      "conv4/conv4_3/biases:0 (512,)\n",
      "conv5/conv5_1/weights:0 (3, 3, 512, 512)\n",
      "conv5/conv5_1/biases:0 (512,)\n",
      "conv5/conv5_2/weights:0 (3, 3, 512, 512)\n",
      "conv5/conv5_2/biases:0 (512,)\n",
      "conv5/conv5_3/weights:0 (3, 3, 512, 512)\n",
      "conv5/conv5_3/biases:0 (512,)\n",
      "fc6/weights:0 (512, 4096)\n",
      "fc6/biases:0 (4096,)\n",
      "fc7/weights:0 (4096, 4096)\n",
      "fc7/biases:0 (4096,)\n",
      "fc8/weights:0 (4096, 1000)\n",
      "fc8/biases:0 (1000,)\n",
      "GRU_RNN_cell/rnn/gru_cell/gates/weights:0 (70100, 200)\n",
      "GRU_RNN_cell/rnn/gru_cell/gates/biases:0 (200,)\n",
      "GRU_RNN_cell/rnn/gru_cell/candidate/weights:0 (70100, 100)\n",
      "GRU_RNN_cell/rnn/gru_cell/candidate/biases:0 (100,)\n",
      "Fully-connected/weights:0 (100, 6)\n",
      "Fully-connected/biases:0 (6,)\n",
      "Total number of trainable params:  58725154\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "with tf.Graph().as_default():\n",
    "    inputs = tf.placeholder(dtype=tf.float32, shape=(50, 240, 320, 3), name=\"input\")\n",
    "    y = tf.placeholder(dtype=tf.int32, shape=(1,), name='labels')\n",
    "    with slim.arg_scope([slim.conv2d, slim.fully_connected],\n",
    "                      activation_fn=tf.nn.relu,\n",
    "                      weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),\n",
    "                      weights_regularizer=slim.l2_regularizer(0.0005)):\n",
    "        net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1')\n",
    "        net = slim.max_pool2d(net, [2, 2], scope='pool1')\n",
    "        net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')\n",
    "        net = slim.max_pool2d(net, [2, 2], scope='pool2')\n",
    "        net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')\n",
    "        net = slim.max_pool2d(net, [2, 2], scope='pool3')\n",
    "        net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')\n",
    "        net = slim.max_pool2d(net, [2, 2], scope='pool4')\n",
    "        net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')\n",
    "        net = slim.max_pool2d(net, [2, 2], scope='pool5')\n",
    "        net = slim.fully_connected(net, 4096, scope='fc6')\n",
    "        net = slim.dropout(net, 0.5, scope='dropout6')\n",
    "        net = slim.fully_connected(net, 4096, scope='fc7')\n",
    "        net = slim.dropout(net, 0.5, scope='dropout7')\n",
    "        net = slim.fully_connected(net, 1000, activation_fn=None, scope='fc8')\n",
    "    \n",
    "    size = np.prod(net.get_shape().as_list()[1:])\n",
    "\n",
    "    with tf.variable_scope('GRU_RNN_cell'):\n",
    "        rnn_inputs = tf.reshape(net, (-1, 50, size))\n",
    "        cell = tf.contrib.rnn.GRUCell(100)\n",
    "        init_state = cell.zero_state(1, dtype=tf.float32)\n",
    "        rnn_outputs, _ = tf.nn.dynamic_rnn(cell, rnn_inputs, initial_state=init_state)\n",
    "        output = tf.reduce_mean(rnn_outputs, axis=1)\n",
    "\n",
    "    with tf.name_scope('Dense'):\n",
    "        logits = slim.fully_connected(output, 6, scope=\"Fully-connected\")\n",
    "    count_number_trainable_params()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}