ch02-自然语言和单词的离散表示.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 计算ppmi\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch02/ppmi.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4 1 5 6]\n",
      "{'you': 0, 'say': 1, 'goodbye': 2, 'and': 3, 'i': 4, 'hello': 5, '.': 6}\n",
      "{0: 'you', 1: 'say', 2: 'goodbye', 3: 'and', 4: 'i', 5: 'hello', 6: '.'}\n",
      "covariance matrix\n",
      "[[0 1 0 0 0 0 0]\n",
      " [1 0 1 0 1 1 0]\n",
      " [0 1 0 1 0 0 0]\n",
      " [0 0 1 0 1 0 0]\n",
      " [0 1 0 1 0 0 0]\n",
      " [0 1 0 0 0 0 1]\n",
      " [0 0 0 0 0 1 0]]\n",
      "--------------------------------------------------\n",
      "PPMI\n",
      "[[0.    1.807 0.    0.    0.    0.    0.   ]\n",
      " [1.807 0.    0.807 0.    0.807 0.807 0.   ]\n",
      " [0.    0.807 0.    1.807 0.    0.    0.   ]\n",
      " [0.    0.    1.807 0.    1.807 0.    0.   ]\n",
      " [0.    0.807 0.    1.807 0.    0.    0.   ]\n",
      " [0.    0.807 0.    0.    0.    0.    2.807]\n",
      " [0.    0.    0.    0.    0.    2.807 0.   ]]\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "from common.util import preprocess, create_co_matrix, cos_similarity, ppmi\n",
    "\n",
    "\n",
    "text = 'You say goodbye and I say hello.'\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "\n",
    "print(corpus)\n",
    "print(word_to_id)\n",
    "print(id_to_word)\n",
    "\n",
    "vocab_size = len(word_to_id)\n",
    "C = create_co_matrix(corpus, vocab_size)\n",
    "W = ppmi(C)\n",
    "\n",
    "np.set_printoptions(precision=3)  # 小数点后显示3位有效数字\n",
    "print('covariance matrix')\n",
    "print(C)\n",
    "print('-'*50)\n",
    "print('PPMI')\n",
    "print(W)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 查找类似的单词\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch02/most_similar.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[query] you\n",
      " goodbye: 0.7071067691154799\n",
      " i: 0.7071067691154799\n",
      " hello: 0.7071067691154799\n",
      " say: 0.0\n",
      " and: 0.0\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from common.util import preprocess, create_co_matrix, most_similar\n",
    "\n",
    "\n",
    "text = 'You say goodbye and I say hello.'\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "vocab_size = len(word_to_id)\n",
    "C = create_co_matrix(corpus, vocab_size)\n",
    "\n",
    "most_similar('you', word_to_id, id_to_word, C, top=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 计算单词的余弦相似度\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch02/similarity.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7071067691154799\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from common.util import preprocess, create_co_matrix, cos_similarity\n",
    "\n",
    "\n",
    "text = 'You say goodbye and I say hello.'\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "vocab_size = len(word_to_id)\n",
    "C = create_co_matrix(corpus, vocab_size)\n",
    "\n",
    "c0 = C[word_to_id['you']]  #「you」的单词向量\n",
    "c1 = C[word_to_id['i']]  #「i」的单词向量\n",
    "print(cos_similarity(c0, c1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 绘制单词的特征值\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch02/count_method_small.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-3.409e-01 -1.110e-16 -3.886e-16 -1.205e-01  0.000e+00  9.323e-01\n",
      "   2.226e-16]\n",
      " [ 0.000e+00 -5.976e-01  1.802e-01  0.000e+00 -7.812e-01  0.000e+00\n",
      "   0.000e+00]\n",
      " [-4.363e-01 -4.241e-17 -2.172e-16 -5.088e-01 -1.767e-17 -2.253e-01\n",
      "  -7.071e-01]\n",
      " [-2.614e-16 -4.978e-01  6.804e-01 -6.574e-17  5.378e-01  9.951e-17\n",
      "   1.201e-17]\n",
      " [-4.363e-01 -3.229e-17 -1.654e-16 -5.088e-01 -1.345e-17 -2.253e-01\n",
      "   7.071e-01]\n",
      " [-7.092e-01 -3.229e-17 -1.654e-16  6.839e-01 -1.345e-17 -1.710e-01\n",
      "   5.889e-17]\n",
      " [ 3.056e-16 -6.285e-01 -7.103e-01  8.846e-17  3.169e-01 -2.847e-16\n",
      "  -1.546e-17]]\n",
      "[3.168e+00 3.168e+00 2.703e+00 2.703e+00 1.514e+00 1.514e+00 1.484e-16]\n",
      "[[ 0.000e+00 -5.976e-01 -2.296e-16 -4.978e-01 -1.186e-16  2.145e-16\n",
      "  -6.285e-01]\n",
      " [-3.409e-01 -1.110e-16 -4.363e-01  0.000e+00 -4.363e-01 -7.092e-01\n",
      "   0.000e+00]\n",
      " [ 1.205e-01 -5.551e-16  5.088e-01  0.000e+00  5.088e-01 -6.839e-01\n",
      "   0.000e+00]\n",
      " [-0.000e+00 -1.802e-01 -1.586e-16 -6.804e-01  6.344e-17  9.119e-17\n",
      "   7.103e-01]\n",
      " [-9.323e-01 -5.551e-17  2.253e-01  0.000e+00  2.253e-01  1.710e-01\n",
      "   0.000e+00]\n",
      " [-0.000e+00  7.812e-01  2.279e-16 -5.378e-01  3.390e-16 -2.717e-16\n",
      "  -3.169e-01]\n",
      " [ 0.000e+00  2.632e-16 -7.071e-01  8.043e-18  7.071e-01  9.088e-17\n",
      "   1.831e-17]]\n",
      "[0 1 0 0 0 0 0]\n",
      "[0.    1.807 0.    0.    0.    0.    0.   ]\n",
      "[-3.409e-01 -1.110e-16 -3.886e-16 -1.205e-01  0.000e+00  9.323e-01\n",
      "  2.226e-16]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGuxJREFUeJzt3X90VfWZ7/H3AwkkajkgKkQ0gi1WavhlThS0YisGMq2tUC/+vBREmoXWWbVr6ZIuxrnaOjNYmWu1ZXknWjE6rCsXVHS0UiDWKhVHQg0IKkYURUyjRUkVEkvIc//IThrShCTsw0ni9/NaK+vsvc9z9vNk5+STzT7nqLk7IiISlj7dPYCIiKSfwl9EJEAKfxGRACn8RUQCpPAXEQmQwl9EJEAKfxGRACn8RUQCpPAXEQlQRncP0J7jjjvOhw8f3t1jiIj0Khs3bvyzux/fUV2PDf/hw4dTXl7e3WOIiPQqZvZuZ+p02UdEJEAKfxGRACn8RUQCpPAXEQmQwl9EJEDBhP+OHTvIy8vrdP2tt97KokWLAJg9ezYrVqw4UqNJF51zzjkp3V/L58aDDz7I9ddfn9L9i/REwYS/fHG8+OKL3T2CSK8XVPgfOHCAH/zgB5xxxhlMmTKF2tpatm/fTlFREfn5+Zx33nm88cYbh9xHWVkZ48ePZ/To0cyZM4fPP/88TdNLk/79+3P66adTWFjIFVdcwaJFi6ioqGDChAmMGTOG6dOn88knnwC0u33jxo2MHTuWiRMnsnjx4oP2v3PnToqKivjqV7/KbbfdBsAtt9zC3Xff3VyzYMEC7rnnHgDuvPNOCgoKGDNmDJMmTfq7urvvvpubbrqJvLw8Ro8ezbJlywB47rnnuOiii5prr7/+eh588MHUHzCRNgQV/pWVlfzwhz9k69atDBw4kEcffZTi4mJ++ctfsnHjRhYtWsR1113X7uPr6uqYPXs2y5Yt49VXX6W+vp577703jd+BlJeXU19fzyuvvMJjjz3W/EHA73//+9xxxx1s3ryZ0aNHN4d2e9uvvvpq7rnnHtavX/93PV5++WWWLl1KRUUFy5cvp7y8nGuuuYbS0lIAGhoaeOSRR7jqqqtYvXo1lZWVvPzyy1RUVJCZmdn8x6Sp7qSTTqKiooJNmzaxdu1abrrpJqqqqtJxuETalZJP+JpZEXA30Be4390Xtrq/P/AQkA/sBi5z9x2p6H0or1fVsGpLNbv21JJdt5thuacwbtw4APLz89mxYwcvvvgiM2bMaH7Moc7kt23bxogRIzjttNMAmDVrFosXL+aGG244st+I8PTmXZSuf4+NT/8nbn14tvJjvj1mGN/5znfYu3cve/bs4fzzzwcafy4zZsygpqamU9tnzpzJM88809yrsLCQwYMHA/C9732PdevWccMNNzB48GBeeeUVqqurGT9+PIMHD2b16tWsXr2aUXlj+LSunn1799K/fz8eXf08RzfsY/z48axbt44rrriCvn37MmTIEM4//3w2bNjAgAED0nwURf4mdvibWV9gMVAIvA9sMLMn3f21FmXXAJ+4+1fM7HLgDuCyuL0P5fWqGkqef4dEdiY5iSx27qln737j9aoaRuUk6Nu3L9XV1QwcOJCKiopO7dPdj+TI0o6nN+9i4TPbOLp/Bsf06wvAwme2Hfb+3B0za/f+1vc1rc+dO5cHH3yQP/3pT8yZM6d5X7Ovu4GaU75BIjuTL2VlsH7Nf3H7Xf+HoZl1/OO8uaxevbrNPhkZGTQ0NDSv19XVHfb3JNJVqbjscxbwlru/7e5/BR4BLm5VczFQGi2vACbboX77UmDVlmoS2ZkksjPpY8aXsjLo08dYtaW6uWbAgAGMGDGC5cuXA42/yJs2bWp3n6effjo7duzgrbfeAuDhhx9uPnuUI6d0/Xsc3T+DRHYmJ4wcizccIKvPAX79uzd4+umnOfrooxk0aBAvvPAC8LefSyKRaHP7wIEDSSQSrFu3DoClS5ce1G/NmjV8/PHH1NbWsnLlSs4991wApk+fzqpVq9iwYQNTp04FYOrUqSx5YAnZ7CeRnclfdn/IGeML2Ll5PS9HdZMmTWLZsmUcOHCAjz76iOeff56zzjqLU045hddee43PP/+cmpoaysrK0nVIRVJy2WcYsLPF+vvA2e3VuHu9mdUAg4E/tywys2KgGCA3NzfWULv21JKTyDpoWx8zdu2pPWjb0qVLufbaa7n99tvZv38/l19+OWPHjm1zn1lZWSxZsoQZM2ZQX19PQUEB8+bNizWndKz6L3WccEw/AI4d/jWsT19eWnQNGYkT+FZBkkQiQWlpKfPmzWPfvn2ceuqpLFmyBKDd7UuWLGHOnDkcddRRzUHe5Otf/zozZ87krbfe4sorrySZTALQr18/vvnNbzJw4ED69m38F8iUKVP48sS1LJl/FWD0zz6Kq26+k5HjzuZA5lH07duX6dOns379esaOHYuZ8fOf/5yhQ4cCcOmllzJmzBhGjhzJ+PHj03E4RQCwuJcyzGwGMNXd50brM4Gz3P0fW9RsjWrej9a3RzW729tvMpn0OP9Vz7vWvElNbePZWJOm9R8XnnbY+5X0u/Q/1vOXFj/L/XX72OeZHNX3AO+V3kRJSQlnnnnmEZ+joaGBM888k+XLlzNy5Mjm7a2faw0NDdx57TTm/PM9/OvsKUd8LpGWzGyjuyc7qkvFZZ/3gZNbrJ8EfNBejZllAAng4xT0bldR3hBqavdTU7ufBvfm5aK8IUeyrRwBsybmsvfz+safZUMD6x/6N9b9fA4b/vcPuOSSS9IS/K+99hpf+cpXmDx58kHBDwc/1z7YUcntswoZ9rUCZk5t/Q9gkZ4jFWf+GcCbwGRgF7ABuNLdt7ao+SEw2t3nRS/4fs/dLz3UfuOe+cPB7/YZNjCborwhjMpJxNqndI+md/tU/6WOIQOymDUxl2+PGdbdYzXTc016is6e+ccO/6jZt4Bf0PhWzwfc/V/M7KdAubs/aWZZwMPAeBrP+C9397cPtc9UhL+ISGg6G/4peZ+/u/8G+E2rbf/cYrkOmNH6cSIi0j2C+oSviIg0UviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBihX+Znasma0xs8rodlA7davMbI+ZPRWnn4iIpEbcM//5QJm7jwTKovW23AnMjNlLRERSJG74XwyURsulwLS2ity9DPg0Zi8REUmRuOE/xN2rAKLbE+KPJCIiR1pGRwVmthYY2sZdC1I9jJkVA8UAubm5qd69iIhEOgx/d7+wvfvMrNrMcty9ysxygA/jDOPuJUAJQDKZ9Dj7EhGR9sW97PMkMCtangU8EXN/IiKSBnHDfyFQaGaVQGG0jpklzez+piIzewFYDkw2s/fNbGrMviIiEkOHl30Oxd13A5Pb2F4OzG2xfl6cPiIiklr6hK+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIBihb+ZHWtma8ysMrod1EbNODNbb2ZbzWyzmV0Wp6eIiMQX98x/PlDm7iOBsmi9tX3A9939DKAI+IWZDYzZV0REYogb/hcDpdFyKTCtdYG7v+nuldHyB8CHwPEx+4qISAxxw3+Iu1cBRLcnHKrYzM4C+gHbY/YVEZEYMjoqMLO1wNA27lrQlUZmlgM8DMxy94Z2aoqBYoDc3Nyu7F5ERLqgw/B39wvbu8/Mqs0sx92ronD/sJ26AcDTwD+5+0uH6FUClAAkk0nvaDYRETk8cS/7PAnMipZnAU+0LjCzfsDjwEPuvjxmPxERSYG44b8QKDSzSqAwWsfMkmZ2f1RzKTAJmG1mFdHXuJh9RUQkBnPvmVdXksmkl5eXd/cYIiK9ipltdPdkR3X6hK+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIBihb+ZHWtma8ysMrod1EbNKWa20cwqzGyrmc2L01NEROKLe+Y/Hyhz95FAWbTeWhVwjruPA84G5pvZiTH7iohIDHHD/2KgNFouBaa1LnD3v7r759Fq/xT0FBGRmOIG8RB3rwKIbk9oq8jMTjazzcBO4A53/yBmXxERiSGjowIzWwsMbeOuBZ1t4u47gTHR5Z6VZrbC3avb6FUMFAPk5uZ2dvciItJFHYa/u1/Y3n1mVm1mOe5eZWY5wIcd7OsDM9sKnAesaOP+EqAEIJlMekeziYjI4Yl72edJYFa0PAt4onWBmZ1kZtnR8iDgXGBbzL4iIhJD3PBfCBSaWSVQGK1jZkkzuz+qGQX8t5ltAn4PLHL3V2P2FRGRGDq87HMo7r4bmNzG9nJgbrS8BhgTp4+IiKSW3nYpIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhKgWOFvZsea2Rozq4xuBx2idoCZ7TKzX8XpKSIi8cU9858PlLn7SKAsWm/Pz4Dfx+wnIiIpEDf8LwZKo+VSYFpbRWaWDwwBVsfsJyIiKRA3/Ie4exVAdHtC6wIz6wP8O3BTzF4iIpIiGR0VmNlaYGgbdy3oZI/rgN+4+04z66hXMVAMkJub28ndi4hIV3UY/u5+YXv3mVm1meW4e5WZ5QAftlE2ETjPzK4DjgH6mdln7v53rw+4ewlQApBMJr2z34SIiHRNh+HfgSeBWcDC6PaJ1gXuflXTspnNBpJtBb+IiKRP3Gv+C4FCM6sECqN1zCxpZvfHHU5ERI4Mc++ZV1eSyaSXl5d39xgiIr2KmW1092RHdfqEr4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiJfAMccc0yX6hX+IiIBUviLiPQQ06ZNIz8/nzPOOIOSkhKg8Yx+wYIFjB07lgkTJlBdXQ3AO++8w8SJEykoKOCWW27pci+Fv4hID/HAAw+wceNGysvLueeee9i9ezd79+5lwoQJbNq0iUmTJnHfffcB8KMf/Yhrr72WDRs2MHTo0C73yogzqJkdCywDhgM7gEvd/ZM26g4Ar0ar77n7d+P0FRH5Ini9qoZVW6rZtaeWYQOzeWvVA6xb+wwAO3fupLKykn79+nHRRRcBkJ+fz5o1awD4wx/+wKOPPgrAzJkzufnmm7vUO+6Z/3ygzN1HAmXReltq3X1c9KXgF5HgvV5VQ8nz71BTu5+cRBabXv4DK5/+LUseW8WmTZsYP348dXV1ZGZmYmYA9O3bl/r6+uZ9NG0/HHHD/2KgNFouBabF3J+ISBBWbakmkZ1JIjuTPmb0ra/lmAEJfv/2p7zxxhu89NJLh3z8ueeeyyOPPALA0qVLu9w/bvgPcfcqgOj2hHbqssys3MxeMjP9gRCR4O3aU8uXsv525f305CTMG/jXuRdxyy23MGHChEM+/u6772bx4sUUFBRQU1PT5f7m7ocuMFsLtPVqwgKg1N0Htqj9xN0HtbGPE939AzM7FXgWmOzu29uoKwaKAXJzc/PffffdLn0zIiK9xV1r3qSmdj+J7MzmbU3rPy487bD3a2Yb3T3ZUV2HZ/7ufqG757Xx9QRQbWY5UcMc4MN29vFBdPs28Bwwvp26EndPunvy+OOP72g0EZFeqyhvCDW1+6mp3U+De/NyUd6QtPSPe9nnSWBWtDwLeKJ1gZkNMrP+0fJxwLnAazH7ioj0aqNyEhRPGkEiO5OqmjoS2ZkUTxrBqJxEWvrHeqsnsBD4f2Z2DfAeMAPAzJLAPHefC4wC/sPMGmj8Y7PQ3RX+IhK8UTmJtIV9a7HC3913A5Pb2F4OzI2WXwRGx+kjIiKppU/4iogESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiPcDevXv59re/zdixY8nLy2PZsmX89Kc/paCggLy8PIqLi3F3tm/fzplnntn8uMrKSvLz87vcT+EvItIDrFq1ihNPPJFNmzaxZcsWioqKuP7669mwYQNbtmyhtraWp556ii9/+cskEgkqKioAWLJkCbNnz+5yP4W/iEg3eb2qhrvWvMmNyzdR/pdjeOa3q7n55pt54YUXSCQS/O53v+Pss89m9OjRPPvss2zduhWAuXPnsmTJEg4cOMCyZcu48soru9w7I87gZnYssAwYDuwALnX3T9qoywXuB04GHPiWu++I01tEpDd7vaqGkuffIZGdSU4ii0/7n8R3/tdDHFu7jZ/85CdMmTKFxYsXU15ezsknn8ytt95KXV0dAJdccgm33XYbF1xwAfn5+QwePLjL/eOe+c8Hytx9JFAWrbflIeBOdx8FnAV8GLOviEivtmpLNYnsTBLZmfQxg30fMzjxJfp99RvceOON/PGPfwTguOOO47PPPmPFihXNj83KymLq1Klce+21XH311YfVP9aZP3Ax8I1ouRR4Dri5ZYGZfQ3IcPc1AO7+WcyeIiK93q49teQksprXq955k/+67+fUN8Apxw/g3nvvZeXKlYwePZrhw4dTUFBw0OOvuuoqHnvsMaZMmXJY/c3dD3t4M9vj7gNbrH/i7oNa1UwD5gJ/BUYAa4H57n6gjf0VA8UAubm5+e++++5hzyYi0pPdteZNamr3k8jObN7WtP7jwtM6fPyiRYuoqanhZz/72UHbzWyjuyc7enyHZ/5mthYY2sZdCzqc7m89zgPGA+/R+BrBbODXrQvdvQQoAUgmk4f/V0lEpIcryhtCyfPvAPClrAw+raunpnY/lxWc1OFjp0+fzvbt23n22WcPu3+H4e/uF7Z3n5lVm1mOu1eZWQ5tX8t/H3jF3d+OHrMSmEAb4S8iEopROQmKJ41g1ZZqdu2pZdjAbC4rOIlROYkOH/v444/H7h/3mv+TwCxgYXT7RBs1G4BBZna8u38EXACUx+wrItLrjcpJdCrsj4S47/ZZCBSaWSVQGK1jZkkzux8gurZ/I1BmZq8CBtwXs6+IiMQQ68zf3XcDk9vYXk7ji7xN62uAMXF6iYhI6sS97CMiIofp9aqag675F+UNSdtlIP3nHUREukHTJ3xraveTk8iipnY/Jc+/w+tVNWnpr/AXEekGrT/hu+z26+iz7xNWbalOS39d9hER6QatP+Fb/C/30eDOrj21aemvM38RkW4wbGA2n9bVH7Tt07p6hg3MTkt/hb+ISDcoyhtCTe1+amr30+DevFyUNyQt/RX+IiLdoOkTvonsTKpq6khkZ1I8aUTa3u2ja/4iIt2kN3/CV0REeiGFv4hIgBT+IiIBUviLiARI4S8iEiCFv4hIgBT+IiIBUviLiARI4S8iEiBz9+6eoU1m9hHw7hHa/XHAn4/QvlNJc6aW5kyt3jBnb5gRUjvnKe5+fEdFPTb8jyQzK3f3ZHfP0RHNmVqaM7V6w5y9YUbonjl12UdEJEAKfxGRAIUa/iXdPUAnac7U0pyp1Rvm7A0zQjfMGeQ1fxGR0IV65i8iErQgwt/MjjWzNWZWGd0OaqPmm2ZW0eKrzsym9bQ5o7pcM1ttZq+b2WtmNryHznmgxfF8Mp0zdmXOqHaAme0ys1+lc8aod2een6eY2cboWG41s3k9dM5xZrY+mnGzmV3W02aM6laZ2R4zeyrN8xWZ2TYze8vM5rdxf38zWxbd/99H8nc7iPAH5gNl7j4SKIvWD+Luv3P3ce4+DrgA2AesTu+YHc8ZeQi4091HAWcBH6ZpviadnbO26Zi6+3fTN16zzs4J8DPg92mZ6u91Zs4q4Jzo+Xk2MN/MTkzjjNC5OfcB33f3M4Ai4BdmNrCHzQhwJzAzbVMBZtYXWAz8A/A14Aoz+1qrsmuAT9z9K8BdwB1HbCB3/8J/AduAnGg5B9jWQX0xsLQnzhk9adb1huMJfNZL5swHHgFmA7/qqXO2qB8MvAec2JPnjOo2ASN74ozAN4Cn0jjbROC3LdZ/AvykVc1vgYnRcgaNH/yyIzFPKGf+Q9y9CiC6PaGD+suB/3vEp/p7nZnzNGCPmT1mZq+Y2Z3RGUU6dfZ4ZplZuZm9lO5LaJEO5zSzPsC/AzelebaWOnU8zexkM9sM7ATucPcP0jgjdPH3yMzOAvoB29MwW5Ou/q6n0zAaf3ZN3o+2tVnj7vVADY1/7FPuC/M/cDeztcDQNu5a0MX95ACjafwLnHIpmDMDOA8YT+PZ3zIaz1h/nYr5mqToeOa6+wdmdirwrJm96u4pDYIUzHkd8Bt332lmqRuslVQcT3ffCYyJLvesNLMV7l6dqhkh5b9HDwOz3L0hFbO12HdKZuwGbT3BWr/dsjM1KfGFCX93v7C9+8ys2sxy3L0qelIe6hr5pcDj7r4/5UOSkjnfB15x97ejx6wEJpDi8E/F8Ww6M3X3t83sORr/YKU0/FMw50TgPDO7DjgG6Gdmn7n7oV4f6I45W+7rAzPbSuNJwIqeNqeZDQCeBv7J3V9K5XypmrGbvA+c3GL9JKD1v96aat43swwgAXx8JIYJ5bLPk8CsaHkW8MQhaq+gey75QOfm3AAMMrOm/3DTBcBraZitpQ7nNLNBZtY/Wj4OOJceOKe7X+Xuue4+HLgReCjVwd8JnTmeJ5lZdrQ8iMbjuS1tEzbqzJz9gMdpPI7L0zhbk678rqfbBmCkmY2IjtPlNM7bUsv5/wfwrEcvAKRcul7s6M4vGq+ZlQGV0e2x0fYkcH+LuuHALqBPD5+zENgMvAo8CPTraXMC50TzbYpur+mpx7NF/Wy65wXfzhzPpp/5pui2uIfO+T+B/UBFi69xPWnGaP0F4COglsaz7alpmu9bwJs0/gt4QbTtp8B3o+UsYDnwFvAycOqRmkWf8BURCVAol31ERKQFhb+ISIAU/iIiAVL4i4gESOEvIhIghb+ISIAU/iIiAVL4i4gE6P8DSw9spZVR0boAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from common.util import preprocess, create_co_matrix, ppmi\n",
    "\n",
    "\n",
    "text = 'You say goodbye and I say hello.'\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "vocab_size = len(id_to_word)\n",
    "C = create_co_matrix(corpus, vocab_size, window_size=1)\n",
    "W = ppmi(C)\n",
    "\n",
    "# SVD\n",
    "U, S, V = np.linalg.svd(W)\n",
    "\n",
    "print(U)\n",
    "print(S)\n",
    "print(V)\n",
    "\n",
    "np.set_printoptions(precision=3)  # 小数点后显示3位有效数字\n",
    "print(C[0])\n",
    "print(W[0])\n",
    "print(U[0])\n",
    "\n",
    "# plot\n",
    "for word, word_id in word_to_id.items():\n",
    "    plt.annotate(word, (U[word_id, 0], U[word_id, 1]))\n",
    "plt.scatter(U[:,0], U[:,1], alpha=0.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 下载并展示ptd数据集的一些信息\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch02/show_ptb.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "corpus size: 929589\n",
      "corpus[:30]: [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23\n",
      " 24 25 26 27 28 29]\n",
      "\n",
      "id_to_word[0]: aer\n",
      "id_to_word[1]: banknote\n",
      "id_to_word[2]: berlitz\n",
      "\n",
      "word_to_id['car']: 3856\n",
      "word_to_id['happy']: 4428\n",
      "word_to_id['lexus']: 7426\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from dataset import ptb\n",
    "\n",
    "\n",
    "corpus, word_to_id, id_to_word = ptb.load_data('train')\n",
    "\n",
    "print('corpus size:', len(corpus))\n",
    "print('corpus[:30]:', corpus[:30])\n",
    "print()\n",
    "print('id_to_word[0]:', id_to_word[0])\n",
    "print('id_to_word[1]:', id_to_word[1])\n",
    "print('id_to_word[2]:', id_to_word[2])\n",
    "print()\n",
    "print(\"word_to_id['car']:\", word_to_id['car'])\n",
    "print(\"word_to_id['happy']:\", word_to_id['happy'])\n",
    "print(\"word_to_id['lexus']:\", word_to_id['lexus'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PTB数据集的SVD计算\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch02/count_method_big.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading ptb.train.txt ... \n",
      "Done\n",
      "counting  co-occurrence ...\n",
      "calculating PPMI ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\teimeichi\\deep-learning-from-scratch-2-nlp-notebook\\common\\util.py:139: RuntimeWarning: overflow encountered in long_scalars\n",
      "  pmi = np.log2(C[i, j] * N / (S[j]*S[i]) + eps)\n",
      "C:\\Users\\teimeichi\\deep-learning-from-scratch-2-nlp-notebook\\common\\util.py:139: RuntimeWarning: invalid value encountered in log2\n",
      "  pmi = np.log2(C[i, j] * N / (S[j]*S[i]) + eps)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0% done\n",
      "2.0% done\n",
      "3.0% done\n",
      "4.0% done\n",
      "5.0% done\n",
      "6.0% done\n",
      "7.0% done\n",
      "8.0% done\n",
      "9.0% done\n",
      "10.0% done\n",
      "11.0% done\n",
      "12.0% done\n",
      "13.0% done\n",
      "14.0% done\n",
      "15.0% done\n",
      "16.0% done\n",
      "17.0% done\n",
      "18.0% done\n",
      "19.0% done\n",
      "20.0% done\n",
      "21.0% done\n",
      "22.0% done\n",
      "23.0% done\n",
      "24.0% done\n",
      "25.0% done\n",
      "26.0% done\n",
      "27.0% done\n",
      "28.0% done\n",
      "29.0% done\n",
      "30.0% done\n",
      "31.0% done\n",
      "32.0% done\n",
      "33.0% done\n",
      "34.0% done\n",
      "35.0% done\n",
      "36.0% done\n",
      "37.0% done\n",
      "38.0% done\n",
      "39.0% done\n",
      "40.0% done\n",
      "41.0% done\n",
      "42.0% done\n",
      "43.0% done\n",
      "44.0% done\n",
      "45.0% done\n",
      "46.0% done\n",
      "47.0% done\n",
      "48.0% done\n",
      "49.0% done\n",
      "50.0% done\n",
      "51.0% done\n",
      "52.0% done\n",
      "53.0% done\n",
      "54.0% done\n",
      "55.0% done\n",
      "56.0% done\n",
      "57.0% done\n",
      "58.0% done\n",
      "59.0% done\n",
      "60.0% done\n",
      "61.0% done\n",
      "62.0% done\n",
      "63.0% done\n",
      "64.0% done\n",
      "65.0% done\n",
      "66.0% done\n",
      "67.0% done\n",
      "68.0% done\n",
      "69.0% done\n",
      "70.0% done\n",
      "71.0% done\n",
      "72.0% done\n",
      "73.0% done\n",
      "74.0% done\n",
      "75.0% done\n",
      "76.0% done\n",
      "77.0% done\n",
      "78.0% done\n",
      "79.0% done\n",
      "80.0% done\n",
      "81.0% done\n",
      "82.0% done\n",
      "83.0% done\n",
      "84.0% done\n",
      "85.0% done\n",
      "86.0% done\n",
      "87.0% done\n",
      "88.0% done\n",
      "89.0% done\n",
      "90.0% done\n",
      "91.0% done\n",
      "92.0% done\n",
      "93.0% done\n",
      "94.0% done\n",
      "95.0% done\n",
      "96.0% done\n",
      "97.0% done\n",
      "98.0% done\n",
      "99.0% done\n",
      "100.0% done\n",
      "calculating SVD ...\n",
      "\n",
      "[query] you\n",
      " i: 0.6480227708816528\n",
      " anybody: 0.576170802116394\n",
      " do: 0.5746399760246277\n",
      " we: 0.5665833950042725\n",
      " 'll: 0.5184658765792847\n",
      "\n",
      "[query] year\n",
      " month: 0.6680757403373718\n",
      " last: 0.6347814798355103\n",
      " quarter: 0.6277652978897095\n",
      " next: 0.6267019510269165\n",
      " earlier: 0.6260565519332886\n",
      "\n",
      "[query] car\n",
      " auto: 0.6572036147117615\n",
      " luxury: 0.6493001580238342\n",
      " truck: 0.5363876819610596\n",
      " domestic: 0.5250194668769836\n",
      " cars: 0.5133699178695679\n",
      "\n",
      "[query] toyota\n",
      " motor: 0.7220377922058105\n",
      " motors: 0.6967273354530334\n",
      " nissan: 0.6133007407188416\n",
      " mazda: 0.5750950574874878\n",
      " lexus: 0.567941427230835\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "from common.util import most_similar, create_co_matrix, ppmi\n",
    "from dataset import ptb\n",
    "\n",
    "\n",
    "window_size = 2\n",
    "wordvec_size = 100\n",
    "\n",
    "corpus, word_to_id, id_to_word = ptb.load_data('train')\n",
    "vocab_size = len(word_to_id)\n",
    "print('counting  co-occurrence ...')\n",
    "C = create_co_matrix(corpus, vocab_size, window_size)\n",
    "print('calculating PPMI ...')\n",
    "W = ppmi(C, verbose=True)\n",
    "\n",
    "print('calculating SVD ...')\n",
    "try:\n",
    "    # truncated SVD (fast!)\n",
    "    from sklearn.utils.extmath import randomized_svd\n",
    "    U, S, V = randomized_svd(W, n_components=wordvec_size, n_iter=5,\n",
    "                             random_state=None)\n",
    "except ImportError:\n",
    "    # SVD (slow)\n",
    "    U, S, V = np.linalg.svd(W)\n",
    "\n",
    "word_vecs = U[:, :wordvec_size]\n",
    "\n",
    "querys = ['you', 'year', 'car', 'toyota']\n",
    "for query in querys:\n",
    "    most_similar(query, word_to_id, id_to_word, word_vecs, top=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}