data-preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load needed packages\n",
    "import pandas as pd\n",
    "import random \n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#make directory for data \n",
    "DATA_PATH = 'data/'\n",
    "os.makedirs(DATA_PATH, exist_ok=True)\n",
    "RAW_DATA = f'{DATA_PATH}raw_data/'\n",
    "os.makedirs(RAW_DATA, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Loading Dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. Download Dataset\n",
    "First we download and extract the gigaword dataset (~3M) [here](https://drive.google.com/file/d/0B6N7tANPyVeBNmlSX19Ld2xDU1E/view?usp=sharing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  277M    0  277M    0     0   277M      0 --:--:--  0:00:01 --:--:--  194M\n"
     ]
    }
   ],
   "source": [
    "#download data\n",
    "!curl --header \"Host: doc-0c-3o-docs.googleusercontent.com\" --header \"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36\" --header \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\" --header \"Accept-Language: en-US,en;q=0.9\" --header \"Cookie: AUTH_s4a6oitvorbtivfcjm2pefm907l1ntir=08690680304265769485|1531648800000|pm1jthdhng09cikkb0pkdcqlod4d76p8\" --header \"Connection: keep-alive\" \"https://doc-0c-3o-docs.googleusercontent.com/docs/securesc/d1n9duui70mcvt9ph3953bv4foh1d3fm/pb1h3k6beg14nfv16poorm9mr6bl90e3/1531656000000/03129501499031348422/08690680304265769485/0B6N7tANPyVeBNmlSX19Ld2xDU1E?e=download\" -o \"summary.tar.gz\" -L\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#extract files \n",
    "!tar -xzf summary.tar.gz -C {RAW_DATA} && mv summary.tar.gz {RAW_DATA}\n",
    "!gunzip {RAW_DATA}sumdata/train/*.*.txt.gz "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2. Import Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#join train sentence pairs together into dataframe\n",
    "data = pd.concat([pd.read_csv(f'{RAW_DATA}sumdata/train/train.article.txt', sep=\"\\n\"), \n",
    "                  pd.read_csv(f'{RAW_DATA}sumdata/train/train.title.txt', sep=\"\\n\")], axis=1)\n",
    "data.columns = [\"article\", \"title\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#join validation sentence pairs together into dataframe\n",
    "val = pd.concat([pd.read_csv(f'{RAW_DATA}sumdata/train/valid.article.filter.txt', sep=\"\\n\"), \n",
    "                  pd.read_csv(f'{RAW_DATA}sumdata/train/valid.title.filter.txt', sep=\"\\n\")], axis=1)\n",
    "val.columns = [\"article\", \"title\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3803956, 189650)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data), len(val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Preprocessing Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1. Looking at the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'new zealand share prices closed #.## percent higher monday in subdued trading ahead of a us holiday , dealers said .'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'new zealand stocks close #.## percent higher'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"kenyan police have a mounted a ##-hour patrol in a village where us presidential candidate barack obama 's grandmother lives after robbers made a botched attempt to rob her solar panel , an official said .\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"kenyan police offer obama 's grandmother security after robbery\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'an israeli military helicopter with a two-man crew crashed near the northern town of afula and burst into flames on wednesday , army radio said .'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'military helicopter crashes in israel'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"bolivian president evo morales on wednesday declared the us ambassador to la paz `` persona non grata , '' accusing the envoy of encouraging the breakup of the country by promoting separatism .\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'bolivia president orders us envoy expelled'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'the algerian cabinet chaired by president abdelaziz bouteflika on sunday adopted the #### finance bill predicated on an oil price of ## dollars a barrel and a growth rate of #.# percent , it was announced here .'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'algeria adopts #### finance bill with oil put at ## dollars a barrel'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    r = random.randint(0,50)\n",
    "    display(data.iloc[r]['article'])\n",
    "    display(data.iloc[r]['title'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save data in tabular format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROCESSED_DATA_PATH = f'{DATA_PATH}processed_data/'\n",
    "os.makedirs(PROCESSED_DATA_PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save train, val datasets \n",
    "data.to_csv(f'{PROCESSED_DATA_PATH}train_ds.csv', index=None)\n",
    "val.to_csv(f'{PROCESSED_DATA_PATH}valid_ds.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Sample Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAMPLE_DATA_PATH = f'{DATA_PATH}sample_data/'\n",
    "os.makedirs(SAMPLE_DATA_PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_train = data.sample(80000)\n",
    "sample_val = val.sample(20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(80000, 20000)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sample_train), len(sample_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save sample train, val, and test datasets \n",
    "sample_train.to_csv(f'{SAMPLE_DATA_PATH}train_ds.csv', index=None)\n",
    "sample_val.to_csv(f'{SAMPLE_DATA_PATH}valid_ds.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Smaller Sample Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_train_ = data.sample(64)\n",
    "sample_val_ = val.sample(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64, 16)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sample_train_), len(sample_val_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save sample train, val, and test datasets \n",
    "sample_train_.to_csv(f'{SAMPLE_DATA_PATH}train_ds_.csv', index=None)\n",
    "sample_val_.to_csv(f'{SAMPLE_DATA_PATH}valid_ds_.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "DATA_PATH = 'data/'\n",
    "PROCESSED_DATA_PATH = f'{DATA_PATH}processed_data/'\n",
    "data = pd.read_csv(f'{PROCESSED_DATA_PATH}train_ds.csv')\n",
    "val = pd.read_csv(f'{PROCESSED_DATA_PATH}valid_ds.csv')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}