6.6 门控循环神经网络从零实现.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 判断是否gpu可用,如果可用设置gpu使用显存\n",
    "if tf.test.is_gpu_available():\n",
    "    gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "    for gpu in gpus:\n",
    "        tf.config.experimental.set_memory_growth(gpu,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 周杰伦歌词数据:\n",
    "链接: https://pan.baidu.com/s/1QieFe3iuDlDeyTYe4dRySA 提取码: vwvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取数据\n",
    "import random\n",
    "import zipfile\n",
    "\n",
    "with zipfile.ZipFile('./data/jaychou_lyrics.txt.zip') as zin:\n",
    "    with zin.open('jaychou_lyrics.txt') as f:\n",
    "        corpus_chars = f.read().decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_chars = corpus_chars.replace('\\n', ' ').replace('\\r', ' ')\n",
    "corpus_chars = corpus_chars[0:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 建立字符索引\n",
    "idx_to_char = list(set(corpus_chars))\n",
    "char_to_idx = dict([(char, i) for i, char in enumerate(idx_to_char)])\n",
    "vocab_size = len(char_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chars: 想要有直升机 想要和你飞到宇宙去 想要和\n",
      "indices: [704, 718, 433, 786, 122, 851, 25, 704, 718, 518, 201, 171, 1017, 664, 694, 220, 25, 704, 718, 518]\n"
     ]
    }
   ],
   "source": [
    "corpus_indices = [char_to_idx[char] for char in corpus_chars]\n",
    "sample = corpus_indices[:20]\n",
    "print('chars:', ''.join([idx_to_char[idx] for idx in sample]))\n",
    "print('indices:', sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=4, shape=(2, 1027), dtype=float32, numpy=\n",
       "array([[1., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 1., ..., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one-hot向量\n",
    "tf.one_hot(np.array([0, 2]), vocab_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, TensorShape([2, 1027]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def to_onehot(X, size): \n",
    "    # X shape: (batch), output shape: (batch, n_class)\n",
    "    return [tf.one_hot(x, size,dtype=tf.float32) for x in X.T]\n",
    "X = np.arange(10).reshape((2, 5))\n",
    "inputs = to_onehot(X, vocab_size)\n",
    "len(inputs), inputs[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从零实现 rnn\n",
    "# 初始化模型参数，我们依据第二章的结论，使用 Ht = theta(concate(Xt + Ht-1, axis=-1) * W),所以可训练的模型参数是 W，输出矩阵Wo及bh,bo\n",
    "\n",
    "# 初始化参数有：Ht-1 的维度\n",
    "vocab_size = 1027\n",
    "num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size\n",
    "class Rnn(object):\n",
    "    def __init__(self, hiden_dim, params=None):\n",
    "        \"\"\"\n",
    "        初始化待训练的参数 variabel\n",
    "        params = (hidden_weight, weight_out, bn_h, bn_o)\n",
    "        \"\"\"\n",
    "        # 是否加载预训练参数\n",
    "        if params:\n",
    "            self.hidden_weight, self.weight_out, self.bn_h, self.bn_o = params\n",
    "        else:\n",
    "            self.hidden_weight = self._ones(shape=(vocab_size+ num_hiddens, num_hiddens))\n",
    "            self.weight_out = self._ones(shape=(num_hiddens, vocab_size))\n",
    "            self.bn_h = tf.Variable(tf.zeros([1,num_hiddens]), dtype=tf.float32)\n",
    "            self.bn_o = tf.Variable(tf.zeros([1,vocab_size]), dtype=tf.float32)\n",
    "        \n",
    "    def _ones(self,shape):\n",
    "        return tf.Variable(tf.random.normal(shape=shape,stddev=0.01,mean=0,dtype=tf.float32))\n",
    "    \n",
    "    def net(self,inputs,Ht):\n",
    "        # 展开做循环计算\n",
    "        outputs = []\n",
    "        for X in inputs:\n",
    "            x = tf.reshape(X,(-1, vocab_size))\n",
    "            # Ht = tf.tanh(theta(concate(Xt + Ht-1, axis=-1) * W) + bn)\n",
    "            Ht = tf.tanh(tf.matmul(tf.concat([X,Ht], axis=1), self.hidden_weight) + self.bn_h)\n",
    "            # 计算输出\n",
    "            Y= tf.matmul(Ht,self.weight_out) + self.bn_o\n",
    "            # 存储中间Y的输出,一般的rnn这里设置return_sequences和return_state参数, \n",
    "            # return_sequence=True返回多个序列,return_state=True代表返回隐向量\n",
    "            outputs.append(Y)\n",
    "\n",
    "        return outputs, Ht"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从零实现 GRU，其实与 RNN 的主要区别在于有门的设计\n",
    "# 确定哪些是需要初始化模型参数：\n",
    "# 核心公式就是一下 四个\n",
    "# Rt=σ(XtWxr+Ht−1Whr+br), Zt=σ(XtWxz+Ht−1Whz+bz)， H~t=tanh(XtWxh+(Rt⊙Ht−1)Whh+bh)， Ht=Zt⊙Ht−1+(1−Zt)⊙H~t\n",
    "\n",
    "# 初始化参数有：Ht-1 的维度\n",
    "vocab_size = 1027\n",
    "num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size\n",
    "class GRU(object):\n",
    "    def __init__(self, hiden_dim, params=None):\n",
    "        \"\"\"\n",
    "        初始化待训练的参数 variabel\n",
    "        params = (Wxr, Whr, br, Wxz, Whz, bz, Wxh, Whh, bh)\n",
    "        \"\"\"\n",
    "        # 是否加载预训练参数\n",
    "        if params:\n",
    "            self.Wxr, self.Whr, self.br, self.Wxz, self.Whz, self.bz, self.Wxh, self.Whh, self.bh = params\n",
    "        else:\n",
    "            self.Wxr = self._ones(shape=(vocab_size, num_hiddens))\n",
    "            self.Whr = self._ones(shape=(num_hiddens, num_hiddens))\n",
    "            self.Wxz = self._ones(shape=(vocab_size, num_hiddens))\n",
    "            self.Whz = self._ones(shape=(num_hiddens, num_hiddens))\n",
    "            self.Wxh = self._ones(shape=(vocab_size, num_hiddens))\n",
    "            self.Whh = self._ones(shape=(num_hiddens, num_hiddens))\n",
    "\n",
    "            \n",
    "            self.br = tf.Variable(tf.zeros([1,num_hiddens]), dtype=tf.float32)\n",
    "            self.bz = tf.Variable(tf.zeros([1,num_hiddens]), dtype=tf.float32)\n",
    "            self.bh = tf.Variable(tf.zeros([1,num_hiddens]), dtype=tf.float32)\n",
    "            \n",
    "            self.Whq = self._ones(shape=(num_hiddens, vocab_size))\n",
    "            self.bq = tf.Variable(tf.zeros([1,vocab_size]), dtype=tf.float32)\n",
    "            \n",
    "    def _ones(self,shape):\n",
    "        return tf.Variable(tf.random.normal(shape=shape,stddev=0.01,mean=0,dtype=tf.float32))\n",
    "    \n",
    "    def net(self, inputs, Ht):\n",
    "        # 展开做循环计算\n",
    "        outputs = []\n",
    "        for X in inputs:\n",
    "            x = tf.reshape(X,(-1, vocab_size))\n",
    "            # Rt=σ(XtWxr+Ht−1Whr+br)\n",
    "            Rt = tf.sigmoid(tf.matmul(x, self.Wxr) + tf.matmul(Ht, self.Whr) + self.br)\n",
    "            # Zt=σ(XtWxz+Ht−1Whz+bz)\n",
    "#             Zt = tf.sigmoid(tf.matmul(x, self.Wxz) + tf.matmul(Ht, self.Whz) + self.bz) 也开始用@用作矩阵乘法\n",
    "            Zt = tf.sigmoid(x @ self.Wxz + Ht @ self.Whz + self.bz)\n",
    "            # 计算 H~t=tanh(XtWxh+(Rt⊙Ht−1)Whh+bh)，注意这里的Rt和Zt都是 num_hiddens*num_hiddens 维，才能这样操作\n",
    "#             H_hat = tf.tanh(tf.matmul(x,self.Wxh) + tf.multiply(Rt,Ht)@self.Whh + self.bh)\n",
    "            H_hat = tf.tanh(x@self.Wxh + (Rt * Ht)@self.Whh + self.bh)\n",
    "            # Ht=Zt⊙Ht−1+(1−Zt)⊙H~t \n",
    "            Ht = Zt * Ht + (1-Zt) * H_hat\n",
    "            Y = tf.matmul(Ht, self.Whq) + self.bq\n",
    "            outputs.append(Y)\n",
    "\n",
    "        return outputs, Ht"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化一个 hidden_state\n",
    "def init_rnn_state(batch_size, num_hiddens):\n",
    "    #随机初始化一个初始值\n",
    "    return tf.zeros(shape=(batch_size, num_hiddens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 256)\n",
      "5\n",
      "(2, 1027)\n",
      "5 (2, 1027) (2, 256)\n"
     ]
    }
   ],
   "source": [
    "# 测试\n",
    "state = init_rnn_state(X.shape[0], num_hiddens)\n",
    "print(state.shape)\n",
    "inputs = to_onehot(X, vocab_size)\n",
    "print(len(inputs))# 5\n",
    "print(inputs[0].shape)\n",
    "gru = GRU(num_hiddens)\n",
    "outputs, state_new = gru.net(inputs, state)\n",
    "# print(outputs)\n",
    "print(len(outputs), outputs[0].shape, state_new.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 定义预测函数\n",
    "以下函数基于前缀prefix（含有数个字符的字符串）来预测接下来的num_chars个字符。这个函数稍显复杂，其中我们将循环神经单元rnn设置成了函数参数，这样在后面小节介绍其他循环神经网络时能重复使用这个函数。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 函数说明：用来生成指定长度的字符，为了给后续模块使用，这里将 rnn模型 设为参数传入\n",
    "def predict_rnn(prefix, num_chars, rnn, init_rnn_state,\n",
    "                num_hiddens, vocab_size,idx_to_char, char_to_idx):\n",
    "    \"\"\"\n",
    "    prefix: 前缀\n",
    "    num_chars: 生成的字符长度\n",
    "    rnn: rnn模型类\n",
    "    init_rnn_state: 隐变量初始化函数\n",
    "    num_hiddens: 隐变量维度\n",
    "    vocab_size: 词表大小\n",
    "    \"\"\"\n",
    "    # 初始化 rnn\n",
    "    rnn = rnn(num_hiddens)\n",
    "    # 初始化隐向量\n",
    "    state = init_rnn_state(1, num_hiddens)\n",
    "    # 生成输出字符列表,并将首字符生成\n",
    "    outputs = [char_to_idx[prefix[0]]]\n",
    "    # 循环解码\n",
    "    for t in range(num_chars + len(prefix) -1):\n",
    "        X = tf.convert_to_tensor(to_onehot(np.array([outputs[-1]]), vocab_size),dtype=tf.float32)\n",
    "        X = tf.reshape(X, [1, -1])\n",
    "#         print(X.shape)  # (1, 1027)\n",
    "        Y, state_new = rnn.net([X], state)\n",
    "        # 添加输出到 outputs 中\n",
    "        if t < len(prefix) -1:\n",
    "            outputs.append(char_to_idx[prefix[t + 1]])\n",
    "        else:\n",
    "            outputs.append(int(np.array(tf.argmax(Y[0],axis=-1))))\n",
    "    print('outputs', outputs)\n",
    "    return ''.join([idx_to_char[i] for i in outputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "outputs [174, 283, 923, 734, 723, 398, 1020, 967, 1025, 364, 743, 930]\n",
      "分开好习碗甩加小纳扫节涯\n"
     ]
    }
   ],
   "source": [
    "# 简单测试一下\n",
    "print(predict_rnn('分开', 10, GRU, init_rnn_state, num_hiddens, vocab_size,\n",
    "            idx_to_char, char_to_idx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 循环神经网络中较容易出现梯度衰减或梯度爆炸。我们会在6.6节（通过时间反向传播）中解释原因。为了应对梯度爆炸，我们可以裁剪梯度（clip gradient）。假设我们把所有模型参数梯度的元素拼接成一个向量 gg，并设裁剪的阈值是θθ。裁剪后的梯度的L2范数不超过θ\n",
    "min((θ / ∥g∥),1)g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_clipping(grads,theta):\n",
    "    norm = np.array([0])\n",
    "    for i in range(len(grads)):\n",
    "        norm+=tf.math.reduce_sum(grads[i] ** 2)\n",
    "    #print(\"norm\",norm)\n",
    "    norm = np.sqrt(norm).item()\n",
    "    new_gradient=[]\n",
    "    if norm > theta:\n",
    "        for grad in grads:\n",
    "            new_gradient.append(grad * theta / norm)\n",
    "    else:\n",
    "        for grad in grads:\n",
    "            new_gradient.append(grad)  \n",
    "    #print(\"new_gradient\",new_gradient)\n",
    "    return new_gradient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 定义模型训练函数，跟之前章节的模型训练函数相比，这里的模型训练函数有以下几点不同：\n",
    "1、使用困惑度评价模型。\n",
    "2、在迭代模型参数前裁剪梯度。4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (tf2-1)",
   "language": "python",
   "name": "tf2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}