01_pre.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import enum\n",
    "\n",
    "from collections import defaultdict\n",
    "from concurrent.futures import ProcessPoolExecutor\n",
    "from pathlib import Path\n",
    "from scipy.sparse import coo_matrix, dok_matrix, lil_matrix, csr_matrix, bsr_matrix\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Input path\n",
    "Set this to the location of input .csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_d = Path('input')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To line-profile a function:\n",
    "\n",
    "```\n",
    "conda install -c anaconda line_profiler\n",
    "```\n",
    "\n",
    "Then run:\n",
    "\n",
    "```\n",
    "%lprun -f f1 f2(...)\n",
    "```\n",
    "\n",
    "Both functions `f1` and `f2` can be the same or different ie. your code starts at `f2` but you are only interested in profiling `f1`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext line_profiler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class _H:\n",
    "    '''Hyperparams'''\n",
    "    def __init__(self, **kwargs):\n",
    "        self.__dict__ = kwargs\n",
    "\n",
    "    def __repr__(self):\n",
    "        return str(self.__dict__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "H = _H(\n",
    "    version = '210101b',\n",
    "    max_users = 450000,\n",
    "    max_questions = 13523,\n",
    "    valid_pct = 0.025, # ~2.5M rows\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def categorize(df, cols):\n",
    "    cats_d = {}\n",
    "    for col in cols:\n",
    "        if df[col].dtype.name == 'category':\n",
    "            print(f'{col} already categorized')\n",
    "        else:\n",
    "            df[col] = pd.Categorical(df[col])\n",
    "        cats_d[col] = df[col].cat.categories.values\n",
    "    return cats_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def means_stds(df, cols):\n",
    "    return { col: df[col].mean() for col in cols }, { col: df[col].std() for col in cols }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question_id': array([    0,     1,     2, ..., 13520, 13521, 13522]),\n",
       " 'bundle_id': array([    0,     1,     2, ..., 13520, 13521, 13522]),\n",
       " 'correct_answer': array([0, 1, 2, 3]),\n",
       " 'part': array([1, 2, 3, 4, 5, 6, 7])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question_dtypes = {\n",
    "    'question_id': 'int16',\n",
    "    'bundle_id': 'int16',\n",
    "    'correct_answer': 'int8',\n",
    "    'part': 'int8',\n",
    "    'tags': 'object',\n",
    "}\n",
    "\n",
    "questions_df = pd.read_csv(\n",
    "    in_d / 'questions.csv',\n",
    "    usecols=question_dtypes.keys(),\n",
    "    dtype=question_dtypes,\n",
    ")\n",
    "\n",
    "qcats = categorize(questions_df, ['question_id', 'bundle_id', 'correct_answer', 'part'])\n",
    "qcats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split tags\n",
    "Tag `n` is renumbered to `n+1` so that 0 means \"no tag\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions_df[[f'tag_{_}' for _ in range(6)]] = (questions_df.tags.str.split(expand=True).fillna('-1').astype('int16') + 1).astype('uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13523 entries, 0 to 13522\n",
      "Data columns (total 11 columns):\n",
      " #   Column          Non-Null Count  Dtype   \n",
      "---  ------          --------------  -----   \n",
      " 0   question_id     13523 non-null  category\n",
      " 1   bundle_id       13523 non-null  category\n",
      " 2   correct_answer  13523 non-null  category\n",
      " 3   part            13523 non-null  category\n",
      " 4   tags            13522 non-null  object  \n",
      " 5   tag_0           13523 non-null  uint8   \n",
      " 6   tag_1           13523 non-null  uint8   \n",
      " 7   tag_2           13523 non-null  uint8   \n",
      " 8   tag_3           13523 non-null  uint8   \n",
      " 9   tag_4           13523 non-null  uint8   \n",
      " 10  tag_5           13523 non-null  uint8   \n",
      "dtypes: category(4), object(1), uint8(6)\n",
      "memory usage: 1.4+ MB\n"
     ]
    }
   ],
   "source": [
    "questions_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question_id</th>\n",
       "      <th>bundle_id</th>\n",
       "      <th>correct_answer</th>\n",
       "      <th>part</th>\n",
       "      <th>tags</th>\n",
       "      <th>tag_0</th>\n",
       "      <th>tag_1</th>\n",
       "      <th>tag_2</th>\n",
       "      <th>tag_3</th>\n",
       "      <th>tag_4</th>\n",
       "      <th>tag_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>51 131 162 38</td>\n",
       "      <td>52</td>\n",
       "      <td>132</td>\n",
       "      <td>163</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>131 36 81</td>\n",
       "      <td>132</td>\n",
       "      <td>37</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>131 101 162 92</td>\n",
       "      <td>132</td>\n",
       "      <td>102</td>\n",
       "      <td>163</td>\n",
       "      <td>93</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>131 149 162 29</td>\n",
       "      <td>132</td>\n",
       "      <td>150</td>\n",
       "      <td>163</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>131 5 162 38</td>\n",
       "      <td>132</td>\n",
       "      <td>6</td>\n",
       "      <td>163</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13518</th>\n",
       "      <td>13518</td>\n",
       "      <td>13518</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13519</th>\n",
       "      <td>13519</td>\n",
       "      <td>13519</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13520</th>\n",
       "      <td>13520</td>\n",
       "      <td>13520</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>73</td>\n",
       "      <td>74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13521</th>\n",
       "      <td>13521</td>\n",
       "      <td>13521</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>125</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13522</th>\n",
       "      <td>13522</td>\n",
       "      <td>13522</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>55</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13523 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      question_id bundle_id correct_answer part            tags  tag_0  tag_1  \\\n",
       "0               0         0              0    1   51 131 162 38     52    132   \n",
       "1               1         1              1    1       131 36 81    132     37   \n",
       "2               2         2              0    1  131 101 162 92    132    102   \n",
       "3               3         3              0    1  131 149 162 29    132    150   \n",
       "4               4         4              3    1    131 5 162 38    132      6   \n",
       "...           ...       ...            ...  ...             ...    ...    ...   \n",
       "13518       13518     13518              3    5              14     15      0   \n",
       "13519       13519     13519              3    5               8      9      0   \n",
       "13520       13520     13520              2    5              73     74      0   \n",
       "13521       13521     13521              0    5             125    126      0   \n",
       "13522       13522     13522              3    5              55     56      0   \n",
       "\n",
       "       tag_2  tag_3  tag_4  tag_5  \n",
       "0        163     39      0      0  \n",
       "1         82      0      0      0  \n",
       "2        163     93      0      0  \n",
       "3        163     30      0      0  \n",
       "4        163     39      0      0  \n",
       "...      ...    ...    ...    ...  \n",
       "13518      0      0      0      0  \n",
       "13519      0      0      0      0  \n",
       "13520      0      0      0      0  \n",
       "13521      0      0      0      0  \n",
       "13522      0      0      0      0  \n",
       "\n",
       "[13523 rows x 11 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions_df = questions_df.drop('tags', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "assert np.all(questions_df.isna() == False) # no nans\n",
    "assert np.all(questions_df.values < 2**16) # all fit in int16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "qcols = questions_df.columns.to_list()\n",
    "QCols = enum.IntEnum('QCols', qcols, start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "qcodes_d = {}\n",
    "for col, cats in qcats.items():\n",
    "    # code=0 is reserved for <NA>, NaN and the likes\n",
    "    qcodes_d[col] = { value: code+1 for code, value in enumerate(cats) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "qc_d = {}\n",
    "for row in questions_df.to_numpy():\n",
    "    question_id_code = qcodes_d['question_id'][row[QCols.question_id]]\n",
    "    qc_d[question_id_code] = np.array([\n",
    "        question_id_code,\n",
    "        qcodes_d['bundle_id'][row[QCols.bundle_id]],\n",
    "        qcodes_d['correct_answer'][row[QCols.correct_answer]],\n",
    "        qcodes_d['part'][row[QCols.part]],\n",
    "        row[QCols.tag_0],\n",
    "        row[QCols.tag_1],\n",
    "        row[QCols.tag_2],\n",
    "        row[QCols.tag_3],\n",
    "        row[QCols.tag_4],\n",
    "        row[QCols.tag_5],\n",
    "    ], dtype=np.int16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lectures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "lecture_dtypes = {\n",
    "    'lecture_id': 'int64',\n",
    "    'tag': 'uint8',\n",
    "    'part': 'int8',\n",
    "    'type_of': 'object'\n",
    "}\n",
    "\n",
    "lectures_df = pd.read_csv(\n",
    "    in_d / 'lectures.csv',\n",
    "    usecols=lecture_dtypes.keys(),\n",
    "    dtype=lecture_dtypes,\n",
    ")\n",
    "\n",
    "lcats = categorize(lectures_df, ['lecture_id', 'part', 'type_of'])\n",
    "#lectures_cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.all(lcats['part'] == qcats['part']) # all parts show up on both dfs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tags\n",
    "Tag `n` is renumbered to `n+1` here as well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "lectures_df['tag_0'] = (lectures_df.tag.fillna('-1').astype('int16') + 1).astype('uint8')\n",
    "for i in range(1, 6):\n",
    "    lectures_df[f'tag_{i}']= pd.Series(0, index=lectures_df.index, dtype='uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 418 entries, 0 to 417\n",
      "Data columns (total 10 columns):\n",
      " #   Column      Non-Null Count  Dtype   \n",
      "---  ------      --------------  -----   \n",
      " 0   lecture_id  418 non-null    category\n",
      " 1   tag         418 non-null    uint8   \n",
      " 2   part        418 non-null    category\n",
      " 3   type_of     418 non-null    category\n",
      " 4   tag_0       418 non-null    uint8   \n",
      " 5   tag_1       418 non-null    uint8   \n",
      " 6   tag_2       418 non-null    uint8   \n",
      " 7   tag_3       418 non-null    uint8   \n",
      " 8   tag_4       418 non-null    uint8   \n",
      " 9   tag_5       418 non-null    uint8   \n",
      "dtypes: category(3), uint8(7)\n",
      "memory usage: 28.4 KB\n"
     ]
    }
   ],
   "source": [
    "lectures_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lecture_id</th>\n",
       "      <th>tag</th>\n",
       "      <th>part</th>\n",
       "      <th>type_of</th>\n",
       "      <th>tag_0</th>\n",
       "      <th>tag_1</th>\n",
       "      <th>tag_2</th>\n",
       "      <th>tag_3</th>\n",
       "      <th>tag_4</th>\n",
       "      <th>tag_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>89</td>\n",
       "      <td>159</td>\n",
       "      <td>5</td>\n",
       "      <td>concept</td>\n",
       "      <td>160</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>concept</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>185</td>\n",
       "      <td>45</td>\n",
       "      <td>6</td>\n",
       "      <td>concept</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>192</td>\n",
       "      <td>79</td>\n",
       "      <td>5</td>\n",
       "      <td>solving question</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>317</td>\n",
       "      <td>156</td>\n",
       "      <td>5</td>\n",
       "      <td>solving question</td>\n",
       "      <td>157</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>32535</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>solving question</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>414</th>\n",
       "      <td>32570</td>\n",
       "      <td>113</td>\n",
       "      <td>3</td>\n",
       "      <td>solving question</td>\n",
       "      <td>114</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415</th>\n",
       "      <td>32604</td>\n",
       "      <td>24</td>\n",
       "      <td>6</td>\n",
       "      <td>concept</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416</th>\n",
       "      <td>32625</td>\n",
       "      <td>142</td>\n",
       "      <td>2</td>\n",
       "      <td>concept</td>\n",
       "      <td>143</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>32736</td>\n",
       "      <td>82</td>\n",
       "      <td>3</td>\n",
       "      <td>concept</td>\n",
       "      <td>83</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>418 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    lecture_id  tag part           type_of  tag_0  tag_1  tag_2  tag_3  tag_4  \\\n",
       "0           89  159    5           concept    160      0      0      0      0   \n",
       "1          100   70    1           concept     71      0      0      0      0   \n",
       "2          185   45    6           concept     46      0      0      0      0   \n",
       "3          192   79    5  solving question     80      0      0      0      0   \n",
       "4          317  156    5  solving question    157      0      0      0      0   \n",
       "..         ...  ...  ...               ...    ...    ...    ...    ...    ...   \n",
       "413      32535    8    5  solving question      9      0      0      0      0   \n",
       "414      32570  113    3  solving question    114      0      0      0      0   \n",
       "415      32604   24    6           concept     25      0      0      0      0   \n",
       "416      32625  142    2           concept    143      0      0      0      0   \n",
       "417      32736   82    3           concept     83      0      0      0      0   \n",
       "\n",
       "     tag_5  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  \n",
       "..     ...  \n",
       "413      0  \n",
       "414      0  \n",
       "415      0  \n",
       "416      0  \n",
       "417      0  \n",
       "\n",
       "[418 rows x 10 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lectures_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "lectures_df = lectures_df.drop('tag', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert lectures_df.isna().sum().sum() == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "lcols = lectures_df.columns.to_list()\n",
    "LCols = enum.IntEnum('LCols', lcols, start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "lcodes_d = {}\n",
    "for col, cats in lcats.items():\n",
    "    # code=0 is reserved for <NA>, NaN and the likes\n",
    "    lcodes_d[col] = { value: code+1 for code, value in enumerate(cats) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert max([ max(col_codes.values()) for col_codes in lcodes_d.values() ]) < 2**15 # fit in int16?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc_d = {}\n",
    "for row in lectures_df.to_numpy():\n",
    "    lecture_id_code = lcodes_d['lecture_id'][row[LCols.lecture_id]]\n",
    "    lc_d[lecture_id_code] = np.array([\n",
    "        lecture_id_code,\n",
    "        lcodes_d['part'][row[LCols.part]],\n",
    "        lcodes_d['type_of'][row[LCols.type_of]],\n",
    "        row[LCols.tag_0],\n",
    "        row[LCols.tag_1],\n",
    "        row[LCols.tag_2],\n",
    "        row[LCols.tag_3],\n",
    "        row[LCols.tag_4],\n",
    "        row[LCols.tag_5],\n",
    "    ], dtype=np.int16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 7s, sys: 2.38 s, total: 1min 9s\n",
      "Wall time: 1min 9s\n",
      "CPU times: user 1min 7s, sys: 2.38 s, total: 1min 9s\n",
      "Wall time: 1min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "interaction_dtypes = {\n",
    "    'row_id': 'int32',\n",
    "    'timestamp': 'int64',\n",
    "    'user_id': 'int32',\n",
    "    'content_id': 'int16',\n",
    "    'content_type_id': 'int8',\n",
    "    'task_container_id': 'int16',\n",
    "    'user_answer': 'int8',\n",
    "    'answered_correctly': 'int8',\n",
    "    'prior_question_elapsed_time': 'float32',\n",
    "    'prior_question_had_explanation': 'boolean'\n",
    "}\n",
    "\n",
    "i_df = pd.read_csv(\n",
    "    in_d / 'train.csv', \n",
    "    usecols=interaction_dtypes.keys(),\n",
    "    dtype=interaction_dtypes,\n",
    "    #nrows=10**6,\n",
    ")\n",
    "\n",
    "icats = categorize(i_df, ['task_container_id', 'user_answer', 'answered_correctly', 'prior_question_had_explanation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, True], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icats['prior_question_had_explanation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "icols = i_df.columns.to_list()\n",
    "ICols = enum.IntEnum('ICols', icols, start=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['row_id',\n",
       " 'timestamp',\n",
       " 'user_id',\n",
       " 'content_id',\n",
       " 'content_type_id',\n",
       " 'task_container_id',\n",
       " 'user_answer',\n",
       " 'answered_correctly',\n",
       " 'prior_question_elapsed_time',\n",
       " 'prior_question_had_explanation']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "icodes_d = {}\n",
    "for col, cats in icats.items():\n",
    "    # code=0 is reserved for <NA>, NaN and the likes\n",
    "    icodes_d[col] = { value: code+1 for code, value in enumerate(cats) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hack in <NA> in cats_d['prior_question_had_explanation']\n",
    "icodes_d['prior_question_had_explanation'][pd.NA] = 0\n",
    "icodes_d['prior_question_had_explanation'][np.nan] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_icode = max([ max(col_codes.values()) for col_codes in icodes_d.values() ])\n",
    "assert max_icode < 2**15 # fit in int16?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge all codes into codes_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "codes_d = { **icodes_d, **qcodes_d, **lcodes_d }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_names = sorted([\n",
    "    'already_answered',          # has this question been answered before?\n",
    "    'answered_correctly',        # answered correctly by user\n",
    "    'bundle_id', \n",
    "    'correct_answer', \n",
    "    'lecture_id', \n",
    "    'part', \n",
    "    'qhe',                       # question has explanation (pqhe shifted upwards 1 container)\n",
    "    'question_id', \n",
    "    'task_container_id',\n",
    "    'type_of',                   # lecture type\n",
    "    'user_answer', \n",
    "])\n",
    "\n",
    "# To hide from decoder:\n",
    "# - answered_correctly\n",
    "# - user_answer\n",
    "# - qhe\n",
    "\n",
    "cont_names = sorted([\n",
    "    'attempt_num',               # number of attempts per user_id, question_id\n",
    "    'attempt_num_log',           # log1p of the above\n",
    "    'attempts_correct',          # number of CORRECT attempts per user_id, question_id\n",
    "    'attempts_correct_log',\n",
    "    'attempts_correct_avg',      # attempts_correct / attempts_num\n",
    "    'attempts_correct_avg_log',\n",
    "    'container_ord',             # ordinal of question within container\n",
    "    'qet',                       # question elapsed time (pqet shifted upwards 1 container)\n",
    "    'qet_log',\n",
    "    'qp',                        # probabilty of occurrence of this question\n",
    "    'qp_log',\n",
    "    'timestamp',                 # interaction ts\n",
    "    'timestamp_log',\n",
    "    'tsli',                      # time since last interaction (aka timestamp delta)\n",
    "    'tsli_log',\n",
    "    'clipped_tsli',              # tsli clipped to 20 minutes\n",
    "    'clipped_tsli_log',\n",
    "    'ts_mod_1day',               # timestamp modulus 1 day\n",
    "    'ts_mod_1day_sin',\n",
    "    'ts_mod_1day_cos',\n",
    "    'ts_mod_1week',              # timestamp modulus 1 week\n",
    "    'ts_mod_1week_sin',\n",
    "    'ts_mod_1week_cos',\n",
    "])\n",
    "\n",
    "# To hide from decoder:\n",
    "# - qet\n",
    "# - qet_log\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "Cats = enum.IntEnum('Cats', cat_names, start=0)\n",
    "Conts = enum.IntEnum('Conts', cont_names, start=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encode user_ids\n",
    "This helps coo -> lil_matrix conversion to not freak out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_d = defaultdict(lambda: len(users_d))\n",
    "for user_id in np.sort(i_df.user_id.unique()):\n",
    "    users_d[user_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(users_d.keys()) == 393656\n",
    "assert np.all(np.array(list(users_d.keys())) == np.array(sorted(users_d.keys())))\n",
    "assert users_d[2746] == 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find probabilty of occurrence of each question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.73 s, sys: 428 ms, total: 4.16 s\n",
      "Wall time: 4.16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tmp_q_df = i_df[i_df.content_type_id == 0]\n",
    "qp_d = (tmp_q_df.content_id.value_counts() / len(tmp_q_df)).to_dict()\n",
    "del tmp_q_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `update_questions`, `update_answers`, `get_x` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_questions(df, Col, cat_names, cont_names, qc_d, lc_d, codes_d, QCols, LCols, Cats, Conts, \n",
    "        hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d, last_q_container_d, last_ts, attempt_num, \n",
    "        attempts_correct, qp_d, users_d):\n",
    "    \n",
    "    df_a = df.values\n",
    "    \n",
    "    n_rows = len(df)\n",
    "    \n",
    "    # Prefetch tslis per (user_id, tcid) for better_tsli calculation\n",
    "    # NOTE the keys (user_id, tcid) are NOT encoded\n",
    "    tsli_d = defaultdict(list)\n",
    "    #for i, (_, row) in enumerate(df_d.items()): # SLOW\n",
    "    #for i, (_, row) in enumerate(df.iterrows()): # SUPER SLOW\n",
    "    for i, row in enumerate(df_a):\n",
    "        user_id, tcid, ts = row[Col.user_id], row[Col.task_container_id], row[Col.timestamp]\n",
    "        encoded_user_id = users_d[user_id]\n",
    "        tsli_d[user_id, tcid].append(ts - last_ts[encoded_user_id,0])\n",
    "        last_ts[encoded_user_id,0] = np.int64(ts)\n",
    "        \n",
    "    # average all tslis in the same task container\n",
    "    tsli_d = { k: sum(v)/len(v) for k, v in tsli_d.items() }\n",
    "    \n",
    "    # append df data to history\n",
    "    for i, row in enumerate(df_a):\n",
    "        user_id = row[Col.user_id]\n",
    "        encoded_user_id = users_d[user_id]\n",
    "        user_has_hist = user_id in hist_cat_d\n",
    "        if user_has_hist:\n",
    "            h_cat  = hist_cat_d [user_id] # just shortcuts\n",
    "            h_cont = hist_cont_d[user_id]\n",
    "            h_tags = hist_tags_d[user_id]\n",
    "            h_tagw = hist_tagw_d[user_id]\n",
    "        \n",
    "        cat  = np.zeros(len(cat_names),  dtype=np.int16)\n",
    "        cont = np.full (len(cont_names), np.nan, dtype=np.float32)\n",
    "\n",
    "        # Categorical test data\n",
    "        content_id = row[Col.content_id]\n",
    "        is_question = row[Col.content_type_id] == 0\n",
    "\n",
    "        if is_question:\n",
    "            encoded_question_id = codes_d['question_id'][content_id]\n",
    "            qc_row = qc_d[encoded_question_id]\n",
    "            cat[Cats.bundle_id]        = qc_row[QCols.bundle_id]\n",
    "            cat[Cats.correct_answer]   = qc_row[QCols.correct_answer]\n",
    "            cat[Cats.part]             = qc_row[QCols.part]\n",
    "            cat[Cats.question_id]      = encoded_question_id\n",
    "            cat[Cats.already_answered] = (int)(attempt_num[encoded_user_id, encoded_question_id-1] > 0)\n",
    "            cat[Cats.qhe]              = 0  # question has explanation?, not known yet    \n",
    "        else:\n",
    "            encoded_lecture_id = codes_d['lecture_id'][content_id]\n",
    "            lc_row = lc_d[encoded_lecture_id]\n",
    "            cat[Cats.lecture_id] = encoded_lecture_id\n",
    "            cat[Cats.part]       = lc_row[LCols.part]\n",
    "            cat[Cats.type_of]    = lc_row[LCols.type_of]\n",
    "\n",
    "        tcid = row[Col.task_container_id]\n",
    "        encoded_pqhe = codes_d['prior_question_had_explanation'][row[Col.prior_question_had_explanation]]\n",
    "        encoded_tcid = codes_d['task_container_id'][tcid]\n",
    "        cat[Cats.task_container_id] = encoded_tcid\n",
    "        \n",
    "        # Continuous test data\n",
    "        ts = row[Col.timestamp]\n",
    "        ts_mod_1day = ts % (1000 * 60 * 60 * 24)\n",
    "        ts_mod_1week = ts % (1000 * 60 * 60 * 24 * 7)\n",
    "        pqet = row[Col.prior_question_elapsed_time]\n",
    "        tsli = tsli_d[(user_id, tcid)] if user_has_hist else np.nan\n",
    "        clipped_tsli = min(tsli, 1000 * 60 * 20) # 20 minutes\n",
    "        \n",
    "        cont[Conts.qet]              = np.nan\n",
    "        cont[Conts.timestamp]        = ts\n",
    "        cont[Conts.tsli]             = tsli\n",
    "        cont[Conts.clipped_tsli]     = clipped_tsli\n",
    "        cont[Conts.qet_log]          = np.nan\n",
    "        cont[Conts.timestamp_log]    = np.log1p(ts)\n",
    "        cont[Conts.tsli_log]         = np.log1p(tsli)\n",
    "        cont[Conts.clipped_tsli_log] = np.log1p(clipped_tsli)\n",
    "        cont[Conts.ts_mod_1day]      = ts_mod_1day\n",
    "        cont[Conts.ts_mod_1day_sin]  = np.sin(ts_mod_1day * 2 * np.pi / (1000 * 60 * 60 * 24))\n",
    "        cont[Conts.ts_mod_1day_cos]  = np.cos(ts_mod_1day * 2 * np.pi / (1000 * 60 * 60 * 24))\n",
    "        cont[Conts.ts_mod_1week]     = ts_mod_1week\n",
    "        cont[Conts.ts_mod_1week_sin] = np.sin(ts_mod_1week * 2 * np.pi / (1000 * 60 * 60 * 24 * 7))\n",
    "        cont[Conts.ts_mod_1week_cos] = np.cos(ts_mod_1week * 2 * np.pi / (1000 * 60 * 60 * 24 * 7))\n",
    "        \n",
    "        # container ordinal\n",
    "        if user_has_hist and h_cat[-1,Cats.task_container_id] == encoded_tcid:\n",
    "            cont[Conts.container_ord] = h_cont[-1,Conts.container_ord] + 1\n",
    "        else:\n",
    "            cont[Conts.container_ord] = 0\n",
    "        \n",
    "        if is_question:\n",
    "            # Update qet and qet_log in history (make qet in last bundle skipping lectures = pqet)\n",
    "            if user_id in last_q_container_d and encoded_tcid != last_q_container_d[user_id]:\n",
    "                idx = h_cat[:,Cats.task_container_id] == last_q_container_d[user_id]\n",
    "                h_cat [idx,Cats.qhe]      = encoded_pqhe\n",
    "                h_cont[idx,Conts.qet]     = pqet\n",
    "                h_cont[idx,Conts.qet_log] = np.log1p(pqet)\n",
    "                        \n",
    "            last_q_container_d[user_id] = encoded_tcid\n",
    "            \n",
    "            # Update attempt_num\n",
    "            an = attempt_num     [encoded_user_id, encoded_question_id-1] # np.uint8\n",
    "            ac = attempts_correct[encoded_user_id, encoded_question_id-1] # np.uint8\n",
    "            cont[Conts.attempt_num]              = an\n",
    "            cont[Conts.attempt_num_log]          = np.log1p(an)\n",
    "            \n",
    "            # Update attempts_correct with what we know so far (will be re-updated after we've got the answers)\n",
    "            cont[Conts.attempts_correct]         = ac\n",
    "            cont[Conts.attempts_correct_log]     = np.log1p(ac)\n",
    "            if an != 0:\n",
    "                cont[Conts.attempts_correct_avg]     = ac / an\n",
    "                cont[Conts.attempts_correct_avg_log] = np.log1p(ac / an)\n",
    "\n",
    "            attempt_num[encoded_user_id, encoded_question_id-1] += np.uint8(1)\n",
    "\n",
    "            # question occurrence prob\n",
    "            cont[Conts.qp]              = qp_d[content_id] # qp_d indexes are non-encoded qids\n",
    "            cont[Conts.qp_log]          = np.log1p(cont[Conts.qp])\n",
    "\n",
    "        # Tags and weights\n",
    "        if is_question:\n",
    "            tags = qc_row[[ QCols.tag_0, QCols.tag_1, QCols.tag_2, QCols.tag_3, QCols.tag_4, QCols.tag_5 ]]\n",
    "        else:\n",
    "            tags = lc_row[[ LCols.tag_0, LCols.tag_1, LCols.tag_2, LCols.tag_3, LCols.tag_4, LCols.tag_5 ]]\n",
    "        tags = tags.astype(np.uint8)\n",
    "        tagw = (tags != 0).astype(np.float16)\n",
    "        sums = tagw.sum()\n",
    "        if sums > 0:\n",
    "            tagw /= sums\n",
    "       \n",
    "        # Concat history and new test data\n",
    "        if user_has_hist:\n",
    "            hist_cat_d [user_id] = np.concatenate((h_cat,  np.expand_dims(cat,  0)))\n",
    "            hist_cont_d[user_id] = np.concatenate((h_cont, np.expand_dims(cont, 0)))\n",
    "            hist_tags_d[user_id] = np.concatenate((h_tags, np.expand_dims(tags, 0)))\n",
    "            hist_tagw_d[user_id] = np.concatenate((h_tagw, np.expand_dims(tagw, 0)))\n",
    "        else:\n",
    "            hist_cat_d [user_id] = np.expand_dims(cat,  0)\n",
    "            hist_cont_d[user_id] = np.expand_dims(cont, 0)\n",
    "            hist_tags_d[user_id] = np.expand_dims(tags, 0)\n",
    "            hist_tagw_d[user_id] = np.expand_dims(tagw, 0)\n",
    "\n",
    "    return df.user_id.values\n",
    "\n",
    "\n",
    "def update_answers(prior_user_ids, prior_group_answers_correct, prior_group_responses, \n",
    "        cat_names, cont_names, codes_d, hist_cat_d, hist_cont_d, users_d, attempt_num, attempts_correct):\n",
    "\n",
    "    idx_per_uid_d = defaultdict(int)\n",
    "    for uid in prior_user_ids:\n",
    "        idx_per_uid_d[uid] -= 1\n",
    "\n",
    "    for i, uid in enumerate(prior_user_ids):\n",
    "        h_cat  = hist_cat_d [uid] # just shortcuts\n",
    "        h_cont = hist_cont_d[uid]\n",
    "        \n",
    "        idx = idx_per_uid_d[uid]\n",
    "        idx_per_uid_d[uid] += 1\n",
    "        \n",
    "        # Update categorical vars\n",
    "        h_cat [idx,Cats.answered_correctly] = codes_d['answered_correctly'][prior_group_answers_correct[i]]\n",
    "        h_cat [idx,Cats.user_answer]        = codes_d['user_answer'][prior_group_responses[i]]\n",
    "\n",
    "        # Update continuous vars\n",
    "        eqid = h_cat[idx,Cats.question_id]\n",
    "        if eqid > 0: # it's a question\n",
    "            assert prior_group_answers_correct[i] >= 0\n",
    "            euid = users_d[uid]\n",
    "            ac = attempts_correct[euid,eqid-1] # np.int8\n",
    "            an = h_cont[idx,Conts.attempt_num] # np.float32\n",
    "            h_cont[idx,Conts.attempts_correct]         = ac\n",
    "            h_cont[idx,Conts.attempts_correct_log]     = np.log1p(ac)\n",
    "            if an != 0:\n",
    "                h_cont[idx,Conts.attempts_correct_avg]     = ac / an\n",
    "                h_cont[idx,Conts.attempts_correct_avg_log] = np.log1p(ac / an)\n",
    "\n",
    "            attempts_correct[euid,eqid-1] = ac + np.uint8(prior_group_answers_correct[i])\n",
    "        else:\n",
    "            assert prior_group_answers_correct[i] == -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ```proxy_append_df```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def proxy_append_df(df):\n",
    "    hist_cat_d         = {}\n",
    "    hist_cont_d        = {}\n",
    "    hist_tags_d        = {}\n",
    "    hist_tagw_d        = {}\n",
    "    last_q_container_d = {}\n",
    "    last_ts            = defaultdict(np.int64)\n",
    "    attempt_num        = defaultdict(np.uint8)\n",
    "    attempts_correct   = defaultdict(np.uint8)\n",
    "    chunk_size         = None\n",
    "    Col                = enum.IntEnum('Col', df.columns.tolist(), start=0)\n",
    "\n",
    "    # update questions\n",
    "    prior_user_ids = update_questions(\n",
    "        df, Col, cat_names, cont_names, qc_d, lc_d, codes_d, QCols, LCols, Cats, Conts, \n",
    "        hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d, last_q_container_d, last_ts, \n",
    "        attempt_num, attempts_correct, qp_d, users_d)\n",
    "\n",
    "    # update answers\n",
    "    prior_group_answers_correct = df.answered_correctly.values\n",
    "    prior_group_responses       = df.user_answer.values\n",
    "\n",
    "    update_answers(prior_user_ids, prior_group_answers_correct, prior_group_responses, \n",
    "        cat_names, cont_names, codes_d, hist_cat_d, hist_cont_d, users_d, attempt_num, attempts_correct)\n",
    "    \n",
    "    return (hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d, \n",
    "            last_q_container_d, last_ts, attempt_num, attempts_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test `append_df`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%lprun -f update_answers (\n",
    "#    hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d, \n",
    "#    last_last_q_container_d, last_ts, attempt_num, previous_ac) = proxy_append_df(i_df[:100000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#(hist_cat_d, hist_cont_d, hist_tags_d, hist_tagw_d,\n",
    "# last_last_q_container_d, last_ts, attempt_num, attempts_correct) = proxy_append_df(i_df[:100000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* pytorch.sparse -> 32s and can't parallelize (\"torch sparse tensor has no storage\" error)\n",
    "* scipy.sparse.dok_matrix -> 17.5s\n",
    "* scipy.sparse.lil_matrix -> 15.4s\n",
    "* scipy.sparse.csr_matrix -> minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.set_option('display.max_rows', 1000)\n",
    "#pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interesting user_ids:\n",
    "\n",
    "- 8623 (3 containers x 5 questions)\n",
    "- 124 (1 container, meaningful tsli)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#i_df[i_df.user_id == 8623]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.DataFrame(hist_ord_d[8623], columns=['ordinal'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.DataFrame(hist_cat_d[8623], columns=cat_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# just for the header\n",
    "#pd.DataFrame(hist_cont_d[8623][:1], columns=cont_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_df = pd.DataFrame(hist_cont_d[8623], columns=cont_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_df[test_df.attempt_num > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_cont = np.concatenate(list(v for v in hist_cont_d.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#_=plt.hist(np.log1p(all_cont[:,Conts.tsli]), bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.exp(14) / 1000 / 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.DataFrame(hist_cat_d[124], columns=cat_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.DataFrame(hist_cont_d[115], columns=cont_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = np.linspace(i_df.user_id.min()-1, i_df.user_id.max(), num=1024+1, dtype=np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = i_df.groupby(pd.cut(i_df.user_id, bins))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.7 s, sys: 879 ms, total: 4.58 s\n",
      "Wall time: 4.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "groups = [ dfg.get_group(_) for _ in dfg.groups.keys() ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1024/1024 [12:57<00:00,  1.32it/s] \n"
     ]
    }
   ],
   "source": [
    "with ProcessPoolExecutor() as e:\n",
    "    res = list(tqdm(e.map(proxy_append_df, groups), total=len(groups)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 49.3 s, sys: 1.74 s, total: 51 s\n",
      "Wall time: 51 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "merge_dicts = lambda idx: { k: v for d in [ _[idx] for _ in res ] for k, v in d.items() }\n",
    "cat_d                 = merge_dicts(0)\n",
    "cont_d                = merge_dicts(1)\n",
    "tags_d                = merge_dicts(2)\n",
    "tagw_d                = merge_dicts(3)\n",
    "last_q_container_id_d = merge_dicts(4)\n",
    "last_ts               = merge_dicts(5)\n",
    "attempt_num           = merge_dicts(6)\n",
    "attempts_correct      = merge_dicts(7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dok matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all(v.dtype == np.int64 for v in last_ts.values())\n",
    "assert all(v.dtype == np.uint8 for v in attempt_num.values())\n",
    "assert all(v.dtype == np.uint8 for v in attempts_correct.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_ts_dok          = dok_matrix((H.max_users, 1), dtype=np.int64)\n",
    "attempt_num_dok      = dok_matrix((H.max_users, H.max_questions), dtype=np.uint8)\n",
    "attempts_correct_dok = dok_matrix((H.max_users, H.max_questions), dtype=np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_ts_dok._update(last_ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "attempt_num_dok._update(attempt_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "attempts_correct_dok._update(attempts_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del res  # this barely has an effect\n",
    "#gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert dok -> array or coo\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<450000x1 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 393656 stored elements in Dictionary Of Keys format>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_ts_dok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 82.7 ms, sys: 279 µs, total: 83 ms\n",
      "Wall time: 95.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "last_ts = last_ts_dok.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<450000x13523 sparse matrix of type '<class 'numpy.uint8'>'\n",
       "\twith 86867031 stored elements in Dictionary Of Keys format>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attempt_num_dok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.3 s, sys: 496 ms, total: 16.8 s\n",
      "Wall time: 16.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "attempt_num_coo = attempt_num_dok.tocoo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<450000x13523 sparse matrix of type '<class 'numpy.uint8'>'\n",
       "\twith 86867031 stored elements in Dictionary Of Keys format>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attempts_correct_dok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.2 s, sys: 435 ms, total: 11.7 s\n",
      "Wall time: 11.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "attempts_correct_coo = attempts_correct_dok.tocoo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "304"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del attempt_num, attempts_correct\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(last_q_container_id_d) == len(cat_d) == len(i_df.user_id.unique()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert attempt_num_coo.getnnz() == 86867031"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ts = i_df.groupby('user_id')['timestamp'].max().values\n",
    "np.testing.assert_equal(test_ts, last_ts[:len(test_ts),0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_emb = {\n",
    "    'already_answered': 2,\n",
    "    'answered_correctly': 4,\n",
    "    'bundle_id': 9766,\n",
    "    'correct_answer': 5,\n",
    "    'lecture_id': 419,\n",
    "    'part': 8,\n",
    "    'prior_question_had_explanation': 3,\n",
    "    'question_id': 13524,\n",
    "    'task_container_id': 10001,\n",
    "    'type_of': 5,\n",
    "    'user_answer': 6\n",
    "}\n",
    "\n",
    "emb_dim = {\n",
    "    'already_answered': 1,\n",
    "    'answered_correctly': 3,\n",
    "    'bundle_id': 274,\n",
    "    'correct_answer': 4,\n",
    "    'lecture_id': 47,\n",
    "    'part': 5,\n",
    "    'prior_question_had_explanation': 3,\n",
    "    'question_id': 329,\n",
    "    'task_container_id': 278,\n",
    "    'type_of': 4,\n",
    "    'user_answer': 4\n",
    "}\n",
    "\n",
    "tags_n_emb = 187+2 # [0..max_tag, max_tag+1]. max_tag+1 = empty tag \n",
    "tags_emb_dim = tags_n_emb # emb_sz_rule(tags_n_emb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_cat = np.concatenate(list(cat_d.values()))\n",
    "assert np.isnan(all_cat).sum() == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert tags_d[115].dtype == np.uint8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Means and stds of continuous vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_cont = np.concatenate(list(cont_d.values()))\n",
    "assert all_cont.shape[0] == len(i_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.nanmax(all_cont[:,Conts.attempt_num]) == 82"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#assert np.isnan(all_cont[:,Conts.prior_question_elapsed_time]).sum() == i_df.prior_question_elapsed_time.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "means = np.nanmean(all_cont, axis=0, dtype=np.float64)\n",
    "stds  = np.nanstd (all_cont, axis=0, dtype=np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxs = np.nanmax(all_cont, axis=0)\n",
    "mins = np.nanmin(all_cont, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['attempt_num',\n",
       "  'attempt_num_log',\n",
       "  'attempts_correct',\n",
       "  'attempts_correct_avg',\n",
       "  'attempts_correct_avg_log',\n",
       "  'attempts_correct_log',\n",
       "  'clipped_tsli',\n",
       "  'clipped_tsli_log',\n",
       "  'container_ord',\n",
       "  'qet',\n",
       "  'qet_log',\n",
       "  'qp',\n",
       "  'qp_log',\n",
       "  'timestamp',\n",
       "  'timestamp_log',\n",
       "  'ts_mod_1day',\n",
       "  'ts_mod_1day_cos',\n",
       "  'ts_mod_1day_sin',\n",
       "  'ts_mod_1week',\n",
       "  'ts_mod_1week_cos',\n",
       "  'ts_mod_1week_sin',\n",
       "  'tsli',\n",
       "  'tsli_log'],\n",
       " array([1.71275676e-01, 1.02451578e-01, 6.78789338e-02, 3.60193529e-01,\n",
       "        2.56868295e-01, 4.17399861e-02, 1.62953342e+05, 1.10097465e+01,\n",
       "        4.04873294e-01, 2.59525175e+04, 9.92678915e+00, 2.54023347e-04,\n",
       "        2.53917140e-04, 7.70364365e+09, 2.08774076e+01, 3.85844342e+07,\n",
       "        2.39129433e-01, 1.60690095e-02, 2.58239902e+08, 1.16058160e-01,\n",
       "        5.14945556e-02, 2.00618321e+07, 1.12457744e+01]),\n",
       " array([5.84928336e-01, 2.86078097e-01, 3.70166409e-01, 4.47063082e-01,\n",
       "        3.12723449e-01, 1.84429290e-01, 3.23172954e+05, 1.19666667e+00,\n",
       "        8.40763914e-01, 2.04180510e+04, 8.28739183e-01, 3.84824433e-04,\n",
       "        3.84496934e-04, 1.15926553e+10, 3.33278846e+00, 2.89322120e+07,\n",
       "        7.21060451e-01, 6.50100564e-01, 1.87675544e+08, 7.32093081e-01,\n",
       "        6.69267159e-01, 4.68762026e+08, 1.94163643e+00]),\n",
       " array([8.2000000e+01, 4.4179688e+00, 8.2000000e+01, 1.0000000e+00,\n",
       "        6.9314718e-01, 4.4179688e+00, 1.2000000e+06, 1.3997833e+01,\n",
       "        9.0000000e+00, 3.0000000e+05, 1.2611541e+01, 2.1517295e-03,\n",
       "        2.1494180e-03, 8.7425769e+10, 2.5194056e+01, 8.6400000e+07,\n",
       "        1.0000000e+00, 1.0000000e+00, 6.0480000e+08, 1.0000000e+00,\n",
       "        1.0000000e+00, 8.3884261e+10, 2.5152704e+01], dtype=float32),\n",
       " array([ 0.0000000e+00,  0.0000000e+00,  0.0000000e+00,  0.0000000e+00,\n",
       "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00,  0.0000000e+00,\n",
       "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00,  1.0073405e-08,\n",
       "         1.0073405e-08,  0.0000000e+00,  0.0000000e+00,  0.0000000e+00,\n",
       "        -1.0000000e+00, -1.0000000e+00,  0.0000000e+00, -1.0000000e+00,\n",
       "        -1.0000000e+00,  0.0000000e+00,  0.0000000e+00], dtype=float32))"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cont_names, means, stds, maxs, mins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20061832.085879758"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "means[Conts.tsli]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.abs(means[Conts.timestamp] - 7703643654.326523) < 0.1 # vs 52\n",
    "#assert np.abs(means[Conts.prior_question_elapsed_time] - 25423.844) < 0.1 # vs 52\n",
    "assert np.abs(means[Conts.tsli] - 20061832.08) < 50 # vs 201221 (precise timestamp)\n",
    "assert np.abs(means[Conts.qet] - 2.59525175e+04) < 0.1 # vs 62, TODO: fix bundle_id -> tcid in 52 and get baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.set_option('display.max_rows', 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.DataFrame(all_cont[:100], columns=cont_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.DataFrame(all_cat[:100], columns=cat_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pickle train data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "WTF, can't pickle enums... We'll rebuild them at train/infer\n",
    "```\n",
    "QCols = enum.IntEnum('QCols', meta.qcols, start=0)\n",
    "LCols = enum.IntEnum('LCols', meta.lcols, start=0)\n",
    "Cats  = enum.IntEnum('Cats',  meta.cat_names, start=0)\n",
    "Conts = enum.IntEnum('Conts', meta.cont_names, start=0)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta = _H(\n",
    "    means=means,\n",
    "    stds=stds,\n",
    "    maxs=maxs,\n",
    "    mins=mins,\n",
    "    qc_d=qc_d,\n",
    "    qcats=qcats,\n",
    "    qcols=qcols,\n",
    "    qcodes_d=qcodes_d,\n",
    "    lc_d=lc_d,\n",
    "    lcats=lcats,\n",
    "    lcols=lcols,\n",
    "    lcodes_d=lcodes_d,\n",
    "    codes_d=codes_d,\n",
    "    cat_names=cat_names,\n",
    "    cont_names=cont_names,\n",
    "    icats=icats,\n",
    "    n_emb=n_emb,\n",
    "    emb_dim=emb_dim,\n",
    "    tags_n_emb=tags_n_emb,\n",
    "    tags_emb_dim=tags_emb_dim,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = _H(\n",
    "    cat_d=cat_d,\n",
    "    cont_d=cont_d,\n",
    "    tags_d=tags_d,\n",
    "    tagw_d=tagw_d,\n",
    "    last_q_contained_id_d=last_q_container_id_d,\n",
    "    attempt_num_coo=attempt_num_coo,\n",
    "    attempts_correct_coo=attempts_correct_coo,\n",
    "    last_ts=last_ts,\n",
    "    qp_d=qp_d,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'210101b'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "H.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 4s, sys: 47 s, total: 1min 51s\n",
      "Wall time: 7min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open(in_d / f'data_v{H.version}.pkl', 'wb') as f:\n",
    "    pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 125 ms, sys: 16.6 ms, total: 141 ms\n",
      "Wall time: 201 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open(in_d / f'meta_v{H.version}.pkl', 'wb') as f:\n",
    "    pickle.dump(meta, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "292px",
    "width": "228px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "283px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}