character_level_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import spacy\n",
    "import itertools\n",
    "import pickle\n",
    "import re\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv(\"./data/train.csv\")\n",
    "df2 = pd.read_csv(\"./data/test.csv\")\n",
    "df = pd.concat((df1, df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train vocab 43\n",
      "test vocab 43\n",
      "Combined vocab 45\n",
      "test vocab not in train 2\n"
     ]
    }
   ],
   "source": [
    "splitter = lambda x: list(x)\n",
    "\n",
    "vocab1 = set(list(itertools.chain.from_iterable(list(df1[\"text\"].apply(splitter)))))\n",
    "vocab2 = set(list(itertools.chain.from_iterable(list(df2[\"text\"].apply(splitter)))))\n",
    "vocab = sorted(list(vocab1.union(vocab2)))\n",
    "print(\"train vocab\", len(vocab1))\n",
    "print(\"test vocab\", len(vocab2))\n",
    "print(\"Combined vocab\", len(vocab))\n",
    "print(\"test vocab not in train\", len([i for i in vocab2 if i not in vocab1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(\"./data/character_level/vocab.npy\", np.array(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000,200000,300000,400000,500000,600000,700000,800000,900000,1000000,1100000,1200000,1300000,1400000,1500000,1600000,1700000,1800000,1900000,2000000,"
     ]
    }
   ],
   "source": [
    "f = open(\"../fasttext/crawl-300d-2M-subword.vec\", \"r\", encoding=\"utf-8\")\n",
    "f_out = open(\"../fasttext/hw1_vocab_character.vec\", \"w\")\n",
    "f_out.write(\"{} {}\\n\".format(len(vocab), 300))\n",
    "d = {}\n",
    "for i, line in enumerate(f):\n",
    "    if i == 0:\n",
    "        continue\n",
    "    if i % 100000 == 0:\n",
    "        print(i, end=\",\")\n",
    "    word = line.split(\" \")[0]\n",
    "    if word in vocab:\n",
    "        d[word] = line.strip().split(' ')\n",
    "        f_out.write(line)\n",
    "    if len(d) == len(vocab):\n",
    "        break\n",
    "f.close()\n",
    "f_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(d, open(\"./data/character_level/vocab_ft.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3338, 4)\n",
      "(1084, 4)\n",
      "(4422, 4)\n",
      "number of 1 grams: 45\n",
      "number of 2 grams: 570\n",
      "number of 3 grams: 3393\n",
      "number of 4 grams: 9393\n",
      "number of 5 grams: 16732\n"
     ]
    }
   ],
   "source": [
    "### save grams\n",
    "\n",
    "df1 = pd.read_csv(\"./data/character_level/train.csv\", index_col=\"ID\")\n",
    "df2 = pd.read_csv(\"./data/character_level/test.csv\", index_col=\"ID\")\n",
    "df = pd.concat([df1, df2])\n",
    "print(df1.shape)\n",
    "print(df2.shape)\n",
    "print(df.shape)\n",
    "vocab = np.load(\"./data/character_level/vocab.npy\")\n",
    "\n",
    "def get_ngram(sent, gram):\n",
    "    chars = list(sent)\n",
    "    if len(chars) < gram:\n",
    "        return []\n",
    "    ngrams = []\n",
    "    for i in range(len(chars)-(gram-1)):\n",
    "        ngrams.append(\"_\".join(chars[i:i+gram]))\n",
    "    return ngrams\n",
    "\n",
    "def save_ngram(df, n=2):\n",
    "    all_ngrams = []\n",
    "    for ngrams in df.raw_text.apply(lambda x: get_ngram(x, n)):\n",
    "        all_ngrams.extend(ngrams)\n",
    "    all_ngrams = np.array(sorted(list(set(all_ngrams))))\n",
    "    np.save(\"./data/character_level/{}grams.npy\".format(n), all_ngrams)\n",
    "    print(\"number of {} grams: {}\".format(n, len(all_ngrams)))\n",
    "    return all_ngrams\n",
    "\n",
    "for n in range(1, 6):\n",
    "    save_ngram(df, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import data_utils\n",
    "importlib.reload(data_utils)\n",
    "\n",
    "train_val_data, holdout_test_data, test_data = data_utils.prep_all_data(\n",
    "    path=\"./data/character_level/\",\n",
    "    char_level=True,\n",
    "    train_file=\"train_val.csv\",\n",
    "    val_file=\"holdout_test.csv\",\n",
    "    test_file=\"test.csv\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "vocab = train_val_data.text_field.vocab.itos\n",
    "ft_emb = pickle.load(open(\"./data/character_level/vocab_ft.pkl\", \"rb\"))\n",
    "emb_matrix = []\n",
    "num_unknown = 0\n",
    "for word in vocab:\n",
    "    if word in ft_emb:\n",
    "        vec = ft_emb[word][1:]\n",
    "        emb_matrix.append([float(i) for i in vec])\n",
    "    else:\n",
    "        num_unknown += 1\n",
    "        vec = [np.random.normal() for i in range(300)]\n",
    "        emb_matrix.append(vec)\n",
    "emb_matrix = torch.tensor(emb_matrix)\n",
    "torch.save(emb_matrix, \"./data/character_level/emb_matrix_ft.pt\")\n",
    "num_unknown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import importlib\n",
    "import pickle\n",
    "\n",
    "import data_utils\n",
    "import model_utils\n",
    "import train_utils\n",
    "import evaluation\n",
    "import submission\n",
    "importlib.reload(data_utils)\n",
    "importlib.reload(model_utils)\n",
    "importlib.reload(train_utils)\n",
    "importlib.reload(evaluation)\n",
    "importlib.reload(submission)\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PATH = \"./data/character_level/\"\n",
    "NGRAM = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, val_data, test_data = data_utils.prep_all_data(path=PATH, ngram=NGRAM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, LR: 0.01, Train Loss: 384.6906, Val Loss: 513.0171, Val f1 0.108\n",
      "Epoch: 5, LR: 0.01, Train Loss: 274.7169, Val Loss: 266.3980, Val f1 0.124\n",
      "Epoch: 10, LR: 0.01, Train Loss: 267.0656, Val Loss: 266.5909, Val f1 0.121\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(model_utils)\n",
    "importlib.reload(data_utils)\n",
    "\n",
    "m = model_utils.BaseModelNGram(path=PATH, ngram=NGRAM)\n",
    "result = train_utils.train(train_data, val_data, m,\n",
    "                          lr=1e-2, print_freq=5, max_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, LR: 0.01, Train Loss: 366.7863, Val Loss: 297.4173, Val f1 0.102\n",
      "Epoch: 5, LR: 0.01, Train Loss: 279.2007, Val Loss: 285.6728, Val f1 0.124\n",
      "Epoch: 10, LR: 0.01, Train Loss: 275.5213, Val Loss: 281.4061, Val f1 0.121\n",
      "Epoch: 15, LR: 0.01, Train Loss: 277.9197, Val Loss: 293.3754, Val f1 0.121\n",
      "Epoch: 20, LR: 0.01, Train Loss: 273.6257, Val Loss: 294.2516, Val f1 0.117\n",
      "Epoch: 25, LR: 0.001, Train Loss: 259.8817, Val Loss: 282.6400, Val f1 0.114\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(model_utils)\n",
    "importlib.reload(data_utils)\n",
    "\n",
    "m = model_utils.GRU(path=PATH)\n",
    "result = train_utils.train(train_data, val_data, m,\n",
    "                          lr=1e-2, print_freq=5, max_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}