examples/00_quick_start/wide_deep_movielens.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<i>Copyright (c) Microsoft Corporation. All rights reserved.</i>\n",
    "\n",
    "<i>Licensed under the MIT License.</i>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wide and Deep Model for Movie Recommendation\n",
    "\n",
    "<br>\n",
    "\n",
    "A linear model with a wide set of crossed-column (co-occurrence) features can memorize the feature interactions, while deep neural networks (DNN) can generalize the feature patterns through low-dimensional dense embeddings learned for the sparse features. [**Wide-and-deep**](https://arxiv.org/abs/1606.07792) learning jointly trains wide linear model and deep neural networks to combine the benefits of memorization and generalization for recommender systems.\n",
    "\n",
    "This notebook shows how to build and test the wide-and-deep model using [TensorFlow high-level Estimator API](https://www.tensorflow.org/api_docs/python/tf/estimator/DNNLinearCombinedRegressor). With the [movie recommendation dataset](https://grouplens.org/datasets/movielens/), we quickly demonstrate following topics:\n",
    "1. How to prepare data\n",
    "2. Build the model\n",
    "3. Use log-hook to estimate performance while training\n",
    "4. Test the model and export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow Version: 1.15.2\n",
      "GPUs:\n",
      " [{'device_name': 'TITAN V', 'total_memory': 12065.375, 'free_memory': 10537.0625}, {'device_name': 'TITAN V', 'total_memory': 12066.6875, 'free_memory': 11137.75}, {'device_name': 'TITAN V', 'total_memory': 12066.6875, 'free_memory': 11137.75}]\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "import math\n",
    "import os\n",
    "from tempfile import TemporaryDirectory\n",
    "\n",
    "import numpy as np\n",
    "import scrapbook as sb\n",
    "import pandas as pd\n",
    "import sklearn.preprocessing\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('ERROR') # only show error messages\n",
    "\n",
    "from reco_utils.common.constants import (\n",
    "    DEFAULT_USER_COL as USER_COL,\n",
    "    DEFAULT_ITEM_COL as ITEM_COL,\n",
    "    DEFAULT_RATING_COL as RATING_COL,\n",
    "    DEFAULT_PREDICTION_COL as PREDICT_COL,\n",
    "    SEED\n",
    ")\n",
    "from reco_utils.common import tf_utils, gpu_utils, plot\n",
    "from reco_utils.dataset import movielens\n",
    "from reco_utils.dataset.pandas_df_utils import user_item_pairs\n",
    "from reco_utils.dataset.python_splitters import python_random_split\n",
    "import reco_utils.evaluation.python_evaluation as evaluator\n",
    "import reco_utils.recommender.wide_deep.wide_deep_utils as wide_deep\n",
    "\n",
    "print(\"Tensorflow Version:\", tf.VERSION)\n",
    "print(\"GPUs:\\n\", gpu_utils.get_gpu_info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "\"\"\"Parameters (papermill)\"\"\"\n",
    "\n",
    "# Recommend top k items\n",
    "TOP_K = 10\n",
    "# Select MovieLens data size: 100k, 1m, 10m, or 20m\n",
    "MOVIELENS_DATA_SIZE = '100k'\n",
    "# Metrics to use for evaluation\n",
    "RANKING_METRICS = [\n",
    "    evaluator.ndcg_at_k.__name__,\n",
    "    evaluator.precision_at_k.__name__,\n",
    "]\n",
    "RATING_METRICS = [\n",
    "    evaluator.rmse.__name__,\n",
    "    evaluator.mae.__name__,\n",
    "]\n",
    "# Use session hook to evaluate model while training\n",
    "EVALUATE_WHILE_TRAINING = True\n",
    "# Item feature column name\n",
    "ITEM_FEAT_COL = 'genres'\n",
    "\n",
    "RANDOM_SEED = SEED  # Set seed for deterministic result\n",
    "\n",
    "# Train and test set pickle file paths. If provided, use them. Otherwise, download the MovieLens dataset.\n",
    "DATA_DIR = None\n",
    "TRAIN_PICKLE_PATH = None\n",
    "TEST_PICKLE_PATH = None\n",
    "EXPORT_DIR_BASE = './outputs/model'\n",
    "# Model checkpoints directory. If None, use temp-dir.\n",
    "MODEL_DIR = None\n",
    "\n",
    "#### Hyperparameters\n",
    "MODEL_TYPE = 'wide_deep'\n",
    "STEPS = 50000  # Number of batches to train\n",
    "BATCH_SIZE = 32\n",
    "# Wide (linear) model hyperparameters\n",
    "LINEAR_OPTIMIZER = 'adagrad'\n",
    "LINEAR_OPTIMIZER_LR = 0.0621  # Learning rate\n",
    "LINEAR_L1_REG = 0.0           # Regularization rate for FtrlOptimizer\n",
    "LINEAR_L2_REG = 0.0\n",
    "LINEAR_MOMENTUM = 0.0         # Momentum for MomentumOptimizer or RMSPropOptimizer\n",
    "# DNN model hyperparameters\n",
    "DNN_OPTIMIZER = 'adadelta'\n",
    "DNN_OPTIMIZER_LR = 0.1\n",
    "DNN_L1_REG = 0.0           # Regularization rate for FtrlOptimizer\n",
    "DNN_L2_REG = 0.0\n",
    "DNN_MOMENTUM = 0.0         # Momentum for MomentumOptimizer or RMSPropOptimizer\n",
    "# Layer dimensions. Defined as follows to make this notebook runnable from Hyperparameter tuning services like AzureML Hyperdrive\n",
    "DNN_HIDDEN_LAYER_1 = 0     # Set 0 to not use this layer\n",
    "DNN_HIDDEN_LAYER_2 = 64    # Set 0 to not use this layer\n",
    "DNN_HIDDEN_LAYER_3 = 128   # Set 0 to not use this layer\n",
    "DNN_HIDDEN_LAYER_4 = 512   # Note, at least one layer should have nodes.\n",
    "DNN_HIDDEN_UNITS = [h for h in [DNN_HIDDEN_LAYER_1, DNN_HIDDEN_LAYER_2, DNN_HIDDEN_LAYER_3, DNN_HIDDEN_LAYER_4] if h > 0]\n",
    "DNN_USER_DIM = 32          # User embedding feature dimension\n",
    "DNN_ITEM_DIM = 16          # Item embedding feature dimension\n",
    "DNN_DROPOUT = 0.8\n",
    "DNN_BATCH_NORM = 1         # 1 to use batch normalization, 0 if not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if MODEL_DIR is None:\n",
    "    TMP_DIR = TemporaryDirectory()\n",
    "    model_dir = TMP_DIR.name\n",
    "else:\n",
    "    if os.path.exists(MODEL_DIR) and os.listdir(MODEL_DIR):\n",
    "        raise ValueError(\n",
    "            \"Model exists in {}. Use different directory name or \"\n",
    "            \"remove the existing checkpoint files first\".format(MODEL_DIR)\n",
    "        )\n",
    "    TMP_DIR = None\n",
    "    model_dir = MODEL_DIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Prepare Data\n",
    "\n",
    "#### 1.1 Movie Rating and Genres Data\n",
    "First, download [MovieLens](https://grouplens.org/datasets/movielens/) data. Movies in the data set are tagged as one or more genres where there are total 19 genres including '*unknown*'. We load *movie genres* to use them as item features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4.81k/4.81k [00:00<00:00, 20.0kKB/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>rating</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>196</td>\n",
       "      <td>242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63</td>\n",
       "      <td>242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>226</td>\n",
       "      <td>242</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>154</td>\n",
       "      <td>242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>306</td>\n",
       "      <td>242</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  rating  genres\n",
       "0     196     242     3.0  Comedy\n",
       "1      63     242     3.0  Comedy\n",
       "2     226     242     5.0  Comedy\n",
       "3     154     242     3.0  Comedy\n",
       "4     306     242     5.0  Comedy"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "use_preset = (TRAIN_PICKLE_PATH is not None and TEST_PICKLE_PATH is not None)\n",
    "if not use_preset:\n",
    "    # The genres of each movie are returned as '|' separated string, e.g. \"Animation|Children's|Comedy\".\n",
    "    data = movielens.load_pandas_df(\n",
    "        size=MOVIELENS_DATA_SIZE,\n",
    "        header=[USER_COL, ITEM_COL, RATING_COL],\n",
    "        genres_col=ITEM_FEAT_COL\n",
    "    )\n",
    "    display(data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2 Encode Item Features (Genres)\n",
    "To use genres from our model, we multi-hot-encode them with scikit-learn's [MultiLabelBinarizer](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MultiLabelBinarizer.html).\n",
    "\n",
    "For example, *Movie id=2355* has three genres, *Animation|Children's|Comedy*, which are being converted into an integer array of the indicator value for each genre like `[0, 0, 1, 1, 1, 0, 0, 0, ...]`. In the later step, we convert this into a float array and feed into the model.\n",
    "\n",
    "> For faster feature encoding, you may load ratings and items separately (by using `movielens.load_item_df`), encode the item-features, then combine the rating and item dataframes by using join-operation. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Genres: ['Action' 'Adventure' 'Animation' \"Children's\" 'Comedy' 'Crime'\n",
      " 'Documentary' 'Drama' 'Fantasy' 'Film-Noir' 'Horror' 'Musical' 'Mystery'\n",
      " 'Romance' 'Sci-Fi' 'Thriller' 'War' 'Western' 'unknown']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>rating</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>196</td>\n",
       "      <td>242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63</td>\n",
       "      <td>242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>226</td>\n",
       "      <td>242</td>\n",
       "      <td>5.0</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>154</td>\n",
       "      <td>242</td>\n",
       "      <td>3.0</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>306</td>\n",
       "      <td>242</td>\n",
       "      <td>5.0</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  rating                                             genres\n",
       "0     196     242     3.0  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "1      63     242     3.0  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "2     226     242     5.0  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "3     154     242     3.0  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4     306     242     5.0  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if not use_preset and ITEM_FEAT_COL is not None:\n",
    "    # Encode 'genres' into int array (multi-hot representation) to use as item features\n",
    "    genres_encoder = sklearn.preprocessing.MultiLabelBinarizer()\n",
    "    data[ITEM_FEAT_COL] = genres_encoder.fit_transform(\n",
    "        data[ITEM_FEAT_COL].apply(lambda s: s.split(\"|\"))\n",
    "    ).tolist()\n",
    "    print(\"Genres:\", genres_encoder.classes_)\n",
    "    display(data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.3 Train and Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75000 train samples and 25000 test samples\n"
     ]
    }
   ],
   "source": [
    "if not use_preset:\n",
    "    train, test = python_random_split(data, ratio=0.75, seed=RANDOM_SEED)\n",
    "else:\n",
    "    train = pd.read_pickle(path=TRAIN_PICKLE_PATH if DATA_DIR is None else os.path.join(DATA_DIR, TRAIN_PICKLE_PATH))\n",
    "    test = pd.read_pickle(path=TEST_PICKLE_PATH if DATA_DIR is None else os.path.join(DATA_DIR, TEST_PICKLE_PATH))\n",
    "    data = pd.concat([train, test])\n",
    "\n",
    "print(\"{} train samples and {} test samples\".format(len(train), len(test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1682 items and 943 users in the dataset\n"
     ]
    }
   ],
   "source": [
    "# Unique items in the dataset\n",
    "if ITEM_FEAT_COL is None:\n",
    "    items = data.drop_duplicates(ITEM_COL)[[ITEM_COL]].reset_index(drop=True)\n",
    "    item_feat_shape = None\n",
    "else:\n",
    "    items = data.drop_duplicates(ITEM_COL)[[ITEM_COL, ITEM_FEAT_COL]].reset_index(drop=True)\n",
    "    item_feat_shape = len(items[ITEM_FEAT_COL][0])\n",
    "# Unique users in the dataset\n",
    "users = data.drop_duplicates(USER_COL)[[USER_COL]].reset_index(drop=True)\n",
    "\n",
    "print(\"Total {} items and {} users in the dataset\".format(len(items), len(users)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Build Model\n",
    "\n",
    "Wide-and-deep model consists of a linear model and DNN. We use the following hyperparameters and feature sets for the model:\n",
    "\n",
    "<br> | <div align=\"center\">Wide (linear) model</div> | <div align=\"center\">Deep neural networks</div>\n",
    "---|---|---\n",
    "Feature set | <ul><li>User-item co-occurrence features<br>to capture how their co-occurrence<br>correlates with the target rating</li></ul> | <ul><li>Deep, lower-dimensional embedding vectors<br>for every user and item</li><li>Item feature vector</li></ul>\n",
    "Hyperparameters | <ul><li>FTRL optimizer</li><li>Learning rate = 0.0029</li><li>L1 regularization = 0.0</li></ul> | <ul><li>Adagrad optimizer</li><li>Learning rate = 0.1</li><li>Hidden units = [128, 256, 32]</li><li>Dropout rate = 0.4</li><li>Use batch normalization (Batch size = 64)</li><li>User embedding vector size = 4</li><li>Item embedding vector size = 4</li></ul>\n",
    "\n",
    "<br>\n",
    "\n",
    "* [FTRL optimizer](https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf)\n",
    "* [Adagrad optimizer](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)\n",
    "\n",
    "Note, the hyperparameters are optimized for the training set. We used **Azure Machine Learning service** ([AzureML](https://azure.microsoft.com/en-us/services/machine-learning-service/)) to find the best hyperparameters, where we further split the training set into two subsets for training and validation respectively so that the test set is being separated from the tuning and training phases. For more details, see [azureml_hyperdrive_wide_and_deep.ipynb](../04_model_select_and_optimize/azureml_hyperdrive_wide_and_deep.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create model checkpoint every n steps. We store the model 5 times.\n",
    "save_checkpoints_steps = max(1, STEPS // 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wide feature specs:\n",
      "\t _VocabularyListCategoricalColumn(key='userID', vocabulary_list=(196, 63, 226, 154, 306, 296, 34, 271 ...\n",
      "\t _VocabularyListCategoricalColumn(key='itemID', vocabulary_list=(242, 302, 377, 51, 346, 474, 265, 46 ...\n",
      "\t _CrossedColumn(keys=(_VocabularyListCategoricalColumn(key='userID', vocabulary_list=(196, 63, 226, 1 ...\n",
      "Deep feature specs:\n",
      "\t _EmbeddingColumn(categorical_column=_VocabularyListCategoricalColumn(key='userID', vocabulary_list=( ...\n",
      "\t _EmbeddingColumn(categorical_column=_VocabularyListCategoricalColumn(key='itemID', vocabulary_list=( ...\n",
      "\t _NumericColumn(key='genres', shape=(19,), default_value=None, dtype=tf.float32, normalizer_fn=None) ...\n"
     ]
    }
   ],
   "source": [
    "# Define wide (linear) and deep (dnn) features\n",
    "wide_columns, deep_columns = wide_deep.build_feature_columns(\n",
    "    users=users[USER_COL].values,\n",
    "    items=items[ITEM_COL].values,\n",
    "    user_col=USER_COL,\n",
    "    item_col=ITEM_COL,\n",
    "    item_feat_col=ITEM_FEAT_COL,\n",
    "    crossed_feat_dim=1000,\n",
    "    user_dim=DNN_USER_DIM,\n",
    "    item_dim=DNN_ITEM_DIM,\n",
    "    item_feat_shape=item_feat_shape,\n",
    "    model_type=MODEL_TYPE,\n",
    ")\n",
    "\n",
    "print(\"Wide feature specs:\")\n",
    "for c in wide_columns:\n",
    "    print(\"\\t\", str(c)[:100], \"...\")\n",
    "print(\"Deep feature specs:\")\n",
    "for c in deep_columns:\n",
    "    print(\"\\t\", str(c)[:100], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmp_dvlimh6', '_tf_random_seed': 42, '_save_summary_steps': 100, '_save_checkpoints_steps': 10000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 5000, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f0e417af128>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "# Build a model based on the parameters\n",
    "model = wide_deep.build_model(\n",
    "    model_dir=model_dir,\n",
    "    wide_columns=wide_columns,\n",
    "    deep_columns=deep_columns,\n",
    "    linear_optimizer=tf_utils.build_optimizer(LINEAR_OPTIMIZER, LINEAR_OPTIMIZER_LR, **{\n",
    "        'l1_regularization_strength': LINEAR_L1_REG,\n",
    "        'l2_regularization_strength': LINEAR_L2_REG,\n",
    "        'momentum': LINEAR_MOMENTUM,\n",
    "    }),\n",
    "    dnn_optimizer=tf_utils.build_optimizer(DNN_OPTIMIZER, DNN_OPTIMIZER_LR, **{\n",
    "        'l1_regularization_strength': DNN_L1_REG,\n",
    "        'l2_regularization_strength': DNN_L2_REG,\n",
    "        'momentum': DNN_MOMENTUM,  \n",
    "    }),\n",
    "    dnn_hidden_units=DNN_HIDDEN_UNITS,\n",
    "    dnn_dropout=DNN_DROPOUT,\n",
    "    dnn_batch_norm=(DNN_BATCH_NORM==1),\n",
    "    log_every_n_iter=max(1, STEPS//10),  # log 10 times\n",
    "    save_checkpoints_steps=save_checkpoints_steps,\n",
    "    seed=RANDOM_SEED\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train and Evaluate Model\n",
    "\n",
    "Now we are all set to train the model. Here, we show how to utilize session hooks to track model performance while training. Our custom hook `tf_utils.evaluation_log_hook` estimates the model performance on the given data based on the specified evaluation functions. Note we pass test set to evaluate the model on rating metrics while we use <span id=\"ranking-pool\">ranking-pool (all the user-item pairs)</span> for ranking metrics.\n",
    "\n",
    "> Note: The TensorFlow Estimator's default loss calculates Mean Squared Error. Square root of the loss is the same as [RMSE](https://en.wikipedia.org/wiki/Root-mean-square_deviation)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = {\n",
    "    'col_user': USER_COL,\n",
    "    'col_item': ITEM_COL,\n",
    "    'col_rating': RATING_COL,\n",
    "    'col_prediction': PREDICT_COL,\n",
    "}\n",
    "\n",
    "# Prepare ranking evaluation set, i.e. get the cross join of all user-item pairs\n",
    "ranking_pool = user_item_pairs(\n",
    "    user_df=users,\n",
    "    item_df=items,\n",
    "    user_col=USER_COL,\n",
    "    item_col=ITEM_COL,\n",
    "    user_item_filter_df=train,  # Remove seen items\n",
    "    shuffle=True,\n",
    "    seed=RANDOM_SEED\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Define training hooks to track performance while training\n",
    "hooks = []\n",
    "if EVALUATE_WHILE_TRAINING:\n",
    "    evaluation_logger = tf_utils.MetricsLogger()\n",
    "    for metrics in (RANKING_METRICS, RATING_METRICS):\n",
    "        if len(metrics) > 0:\n",
    "            hooks.append(\n",
    "                tf_utils.evaluation_log_hook(\n",
    "                    model,\n",
    "                    logger=evaluation_logger,\n",
    "                    true_df=test,\n",
    "                    y_col=RATING_COL,\n",
    "                    eval_df=ranking_pool if metrics==RANKING_METRICS else test.drop(RATING_COL, axis=1),\n",
    "                    every_n_iter=save_checkpoints_steps,\n",
    "                    model_dir=model_dir,\n",
    "                    eval_fns=[evaluator.metrics[m] for m in metrics],\n",
    "                    **({**cols, 'k': TOP_K} if metrics==RANKING_METRICS else cols)\n",
    "                )\n",
    "            )\n",
    "\n",
    "# Define training input (sample feeding) function\n",
    "train_fn = tf_utils.pandas_input_fn(\n",
    "    df=train,\n",
    "    y_col=RATING_COL,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    num_epochs=None,  # We use steps=TRAIN_STEPS instead.\n",
    "    shuffle=True,\n",
    "    seed=RANDOM_SEED,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training steps = 50000, Batch size = 32 (num epochs = 21)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /tmp/tmp_dvlimh6/model.ckpt.\n",
      "INFO:tensorflow:loss = 459.19025, step = 0\n",
      "INFO:tensorflow:global_step/sec: 118.212\n",
      "INFO:tensorflow:loss = 26.569515, step = 5000 (42.298 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10000 into /tmp/tmp_dvlimh6/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 118.699\n",
      "INFO:tensorflow:loss = 25.64497, step = 10000 (87.955 sec)\n",
      "INFO:tensorflow:global_step/sec: 57.0704\n",
      "INFO:tensorflow:loss = 21.546698, step = 15000 (41.779 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20000 into /tmp/tmp_dvlimh6/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 118.598\n",
      "INFO:tensorflow:loss = 39.53526, step = 20000 (88.061 sec)\n",
      "INFO:tensorflow:global_step/sec: 56.9188\n",
      "INFO:tensorflow:loss = 27.464117, step = 25000 (41.943 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 30000 into /tmp/tmp_dvlimh6/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 118.554\n",
      "INFO:tensorflow:loss = 28.321032, step = 30000 (88.207 sec)\n",
      "INFO:tensorflow:global_step/sec: 56.865\n",
      "INFO:tensorflow:loss = 29.131533, step = 35000 (41.896 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmp_dvlimh6/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 118.834\n",
      "INFO:tensorflow:loss = 24.887686, step = 40000 (88.473 sec)\n",
      "INFO:tensorflow:global_step/sec: 56.5719\n",
      "INFO:tensorflow:loss = 22.260078, step = 45000 (41.986 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 50000 into /tmp/tmp_dvlimh6/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 23.359797.\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    \"Training steps = {}, Batch size = {} (num epochs = {})\"\n",
    "    .format(STEPS, BATCH_SIZE, (STEPS*BATCH_SIZE)//len(train))\n",
    ")\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "\n",
    "try:\n",
    "    model.train(\n",
    "        input_fn=train_fn,\n",
    "        hooks=hooks,\n",
    "        steps=STEPS\n",
    "    )\n",
    "except tf.train.NanLossDuringTrainingError:\n",
    "    import warnings\n",
    "    warnings.warn(\n",
    "        \"Training stopped with NanLossDuringTrainingError. \"\n",
    "        \"Try other optimizers, smaller batch size and/or smaller learning rate.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/anaconda/envs/reco_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.0). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "application/papermill.record+json": {
       "eval_ndcg_at_k": [
        0.13612488762996836,
        0.11661381449254485,
        0.09941365302372,
        0.09136117357876819
       ]
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/papermill.record+json": {
       "eval_precision_at_k": [
        0.12629904559915164,
        0.10996818663838814,
        0.09650053022269355,
        0.09013785790031816
       ]
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/papermill.record+json": {
       "eval_rmse": [
        0.9681662763816054,
        0.9582932142420499,
        0.9552528013194066,
        0.9536977992999061
       ]
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/papermill.record+json": {
       "eval_mae": [
        0.7751204778409004,
        0.7647941473007203,
        0.7612266916537285,
        0.7591374960803986
       ]
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnMAAAJQCAYAAADliOKWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XeYFeX5xvHvs7ssvQgsSlNAQATpK6IiGFEEo6IRLMFuLFFiwRjxF6PGxCRqBI1iwSQ2LCA2rIjGiF2W3gRWRFlQihQpUhae3x9n0ONmly1n98wp9+e65tpz5szM3nNcXp+Zd+Ydc3dEREREJDllhB1ARERERCpOxZyIiIhIElMxJyIiIpLEVMyJiIiIJDEVcyIiIiJJTMWciIiISBJTMSciIiKSxFTMiYiIiCQxFXMiIiIiSSwr7ADx0LhxY2/VqlXYMUTSzvTp09e6e07YOaRi1HaKhKO8bWdaFHOtWrUiLy8v7BgiacfMvgw7g1Sc2k6RcJS37VQ3q4iIiEgSUzEnIiIiksRUzImIiIgkMRVzIiIiIklMxZyIiIhIElMxJyIiIpLEVMyJiIiIJDEVcyIiIiJJTMVcYNvOXWzZXhh2DBGRpLJh6w527/awY4ikNRVzgQff/ZybXpofdgwRkaTy22dnM+4TPehDJEwq5gIXH9WGvC/X8ca8b8KOIiKSNEYOOpjRUxbz1bdbw44ikrZUzAVqV89i1OldufHFeazetC3sOCIiSaFtkzr8+ugD+d1zs9XdKhISFXNReh7QkNNzW3DDc3NxV6MkIlIWF/Vpw/bC3epuFQmJirkirj62PV9v3Mb4acvDjiIikhQyM4w7h3RVd6tISFTMFZGdlcHoM7px+xufqVESkVCY2UAzW2Rm+WY2spjP+5rZDDMrNLMhUfO7mdlHZjbfzOaY2RlRnz1qZl+Y2axg6laZmdXdKhIeFXPFOGi/ulx+dFuufXYWu9QoiUgcmVkmMAYYBHQEzjKzjkUW+wo4H3iqyPytwLnu3gkYCNxtZg2iPr/O3bsF06zKzq7uVpFwqJgrwUV9WpNhxsPvLQ07ioikl15AvrsvdfcdwDPA4OgF3H2Zu88BdheZv9jdlwSvVwKrgZz4xFZ3q0hYVMyVICPD+PvQroydupSFX38XdhwRSR/NgeiLdguCeeViZr2AbODzqNm3Bd2vo82semwxi6fuVpH4UzG3Fy0b1mLkoA5cM34W2wt3hR1HRNKDFTOvXFWRmTUFngAucPc9Z+9uADoAhwINgetLWPcSM8szs7w1a9aU59f+QN2tIvGlYq4UQ3u2oGXDWoyasjjsKCKSHgqAllHvWwAry7qymdUDXgVudPeP98x39689YjvwCJHu3P/h7mPdPdfdc3NyKtZDq+5WkfhSMVcKM+Ovv+jM8zNW8OkX68KOIyKpbxrQzsxam1k2cCYwqSwrBsu/ADzu7s8W+axp8NOAU4B5lZq6CHW3isSPirkyaFynOn85tTPXPjuLzdsLw44jIinM3QuB4cBkYCEwwd3nm9mtZnYygJkdamYFwFDgITPb82Dp04G+wPnFDEHypJnNBeYCjYE/V/W+qLtVJD4sHZ50kJub63l5eTFv53cTZ5Nhxt9O61IJqURSn5lNd/fcsHNIxVRG25m/ejNDH/yQl67ow/6NalVSMpHUVt62U2fmyuEPJ3bk/fy1vLVgVdhRRESSgrpbRaqeirlyqFujGncN7cr/vTCXbzdvDzuOiEhSUHerSNVSMVdOh7VpxCndm/N/L8wlHbqoRURipbtbRaqWirkKGHFce5at3cpzM1aEHUVEJCmou1Wk6qiYq4Aa1TIZfUY3/vLaQgrW6yhTRKQsLurThh3qbhWpdCrmKqhjs3r86qjWXPfsHB1lioiUQWaGcedQdbeKVDYVczG4tO+B7Ni1m39/8EXYUUREksKBOepuFalsKuZikJlhjDq9K2PeyWfxqk1hxxERSQrqbhWpXCrmYnRAo9pcd3wHrhk/ix2Fu0tfQUQkzam7VaRyqZirBGf1asm+9Wrwj7eXhB1FRCQpqLtVpPKomKsEZsbfTuvMM9OWM/3L9WHHERFJCupuFakcKuYqSZO6NfjT4E5cO2EWW3cUhh1HRCThqbtVpHKomKtEgzo3pcf++/CX1xaGHUVEJCns6W69bqK6W0UqSsVcJbv55E6889ka/rtoddhRRESSwkV92rBzl7pbRSoqlGLOzAaa2SIzyzezkcV83tfMZphZoZkNiZp/gJlNN7NZZjbfzC6Lb/LS1a9ZjTuGdGHkc3NZv2VH2HFERBKeultFYhP3Ys7MMoExwCCgI3CWmXUssthXwPnAU0Xmfw0c4e7dgMOAkWbWrGoTl9+RbRszqPN+3PjSPNzVbSAiUhp1t4pUXBhn5noB+e6+1N13AM8Ag6MXcPdl7j4H2F1k/g533x68rU4CdxNfP7ADi77ZxKTZK8OOIiKSFNTdKlIxYRRDzYHlUe8LgnllYmYtzWxOsI3b3b3YasnMLjGzPDPLW7NmTUyBK6JGtUxGn96NW19ewNcbv4/77xcRSTbqbhWpmDCKOStmXpnPqbv7cnfvArQFzjOzfUtYbqy757p7bk5OTgWjxqZzi/qcd0QrfjdxjroNRETKQN2tIuUXRjFXALSMet8CKHdfZHBGbj5wVCXlqhKXH30gm7YV8sTH6jYQESkLdbeKlE8Yxdw0oJ2ZtTazbOBMYFJZVjSzFmZWM3i9D3AksKjKklaCrMwMRp3elbvfWsznazaHHUdEJOHt6W69+60l6m4VKYO4F3PuXggMByYDC4EJ7j7fzG41s5MBzOxQMysAhgIPmdn8YPWDgU/MbDbwLvB3d58b730orzY5dbjmuPaMGD+Lnbt2l76CiEiaOzCnDr/up+5WkbII5W5Qd3/N3du7+4Huflsw7yZ3nxS8nubuLdy9trs3cvdOwfwp7t7F3bsGP8eGkb8izul9APVqVmPMO/lhRxGRBBfDWJzdzOyjYBzOOWZ2RtRnrc3sEzNbYmbjg56RhHZhn9bqbhUpg4Qd2iPVmBl3DunKEx99yZyCDWHHEZEEFeNYnFuBc4MD4IHA3WbWIPjsdmC0u7cD1gMXVc0eVB51t4qUjYq5ONqvfg1uPrkT14yfxbadu8KOIyKJKZaxOBe7+5Lg9UpgNZBjZgYcA0wMFn0MOKVqd6NyqLtVpHQq5uLs5K7N6NisPn97/bOwo4hIYoppLM49zKwXkA18DjQCNgTXLFd4m2FRd6vI3qmYC8GfBnfijXnf8P6StWFHEZHEE9NYnABm1hR4ArjA3XeXZ5thD7heHHW3iuydirkQNKiVze1DuvC7ibPZ+P3OsOOISGKJaSxOM6sHvArc6O4fB7PXAg3MLKu0bSbCgOvFUXerSMlUzIWkX/sc+h+8Lze/NC/sKCKSWGIZizMbeAF43N2f3TPf3R14B9hz5+t5wEuVmjoO1N0qUjwVcyG64YQOzC7YyKtzvg47iogkiBjH4jwd6Aucb2azgqlb8Nn1wAgzyydyDd2/4rhblULdrSLFs8gBW2rLzc31vLy8sGMUa+ZX67n48Txeu/IomtSrEXYckUplZtPdPTfsHFIxidp2Pjx1KW8tXMXTF/cmI6O4ywFFklt5206dmQtZ9/334Ze99ud3z80hHQprEZFYqbtV5KdUzCWA3/Rvx7ebd/DUp1+FHUVEJOGpu1Xkp1TMJYBqmRmMPqMrf5+8iGVrt4QdR0Qk4enuVpEfqZhLEG2b1GX4Me0YMWEWhbt2l76CiEiaU3erSISKuQRywRGtqFEtk4emLg07iohIwlN3q0iEirkEkhE0TP96/wvmrdgYdhwRkYSn7lYRFXMJp3mDmtz484MZMWEW23buCjuOiEjC29Pd+sTH6m6V9KRiLgGd2r05B+bU4a43F4UdRUQk4f3Y3bpY3a2SllTMJSAz47ZTOzNp9ko+Xvpt2HFERBLegTl1uPzotupulbSkYi5BNaydzV9/0ZlrJ8xm07adYccREUl46m6VdKViLoEd02Ff+rZvzB9fXhB2FBGRhKfuVklXKuYS3I0/78inX6xj8vxvwo4iIpLw1N0q6UjFXIKrXT2LUad35cYX57F28/aw44iIJDx1t0q6UTGXBHJbNWRIzxaMfG4u7jrSFBHZG3W3SrpRMZckrjm2PSs2fM+zeQVhRxERSXjqbpV0omIuSWRnZTD6jK787Y3PWL5OR5oiIqVRd6ukCxVzSaTDfvW4tG8brp0wm1060hQR2St1t0q6UDGXZH51VBsA/vX+0pCTiIgkPnW3SjpQMZdkMjOMu07vyoPvLuWzb74LO46ISMJTd6ukOhVzSahlw1qMHNiBa8bPZnvhrrDjiIgkNHW3SqpTMZekhua2oHmDmtz91pKwo4iIJDx1t0oqUzGXpMyMv53WmYnTC8hbti7sOCIiCU/drZKqVMwlscZ1qnPbKYcwYsJsNm8vDDuOiFQSMxtoZovMLN/MRhbzeV8zm2FmhWY2pMhnb5jZBjN7pcj8R83sCzObFUzdqno/Eo26WyVVqZhLcgM67Uev1g257dUFYUcRkUpgZpnAGGAQ0BE4y8w6FlnsK+B84KliNnEncE4Jm7/O3bsF06xKipxU1N0qqUjFXAq4+aSOTF28lv98tirsKCISu15AvrsvdfcdwDPA4OgF3H2Zu88Bdhdd2d3fBjbFJWmSurBPawp3u7pbJWWomEsBdWtU467Tu3LD83NZt2VH2HFEJDbNgeVR7wuCeZXhNjObY2ajzax6JW0z6WRmGHcM6aLuVkkZKuZSRO82jTi5azN+/8Jc3NV1IJLErJh5lfGP+gagA3Ao0BC4vthfbnaJmeWZWd6aNWsq4dcmJnW3SipRMZdCrh1wEEvXbOGFmSvCjiIiFVcAtIx63wJYGetG3f1rj9gOPEKkO7e45ca6e6675+bk5MT6axOaulslVaiYSyE1qmUy6oyu3PbqQlZs+D7sOCJSMdOAdmbW2syygTOBSbFu1MyaBj8NOAWYF+s2k526WyVVqJhLMZ2a1efCPq357QR1HYgkI3cvBIYDk4GFwAR3n29mt5rZyQBmdqiZFQBDgYfMbP6e9c3sPeBZoL+ZFZjZ8cFHT5rZXGAu0Bj4c/z2KnGpu1VSQVbYAaTyXdq3DW8vXMWjHy7jwj6tw44jIuXk7q8BrxWZd1PU62lEul+LW/eoEuYfU5kZU8mFfVrzxvxveOLjLznviFZhxxEpN52ZS0FZmRmMOr0b972TT/5qjVAgIrI30d2tX367Jew4IuWmYi5FtWpcm2sHtOea8bPZuet/hqISEZEoe7pbfzdxjrpbJemEUsxV9FE1ZtbNzD4ys/nBWElnxDd5cvllr/1pXCebe99eEnYUEZGEp7tbJVnFvZiL8VE1W4Fz3b0TMBC428waVG3i5GVm3H5aF576dDkzv1ofdhwRkYSm7lZJVmGcmavwo2rcfbG7LwlerwRWA6k9EFKMmtSrwa2DOzFiwmy27igMO45IWjGzW4u8zzSzJ8PKI6VTd6skozCKuUp5VI2Z9QKygc9L+DwtRjEvixM6N6Vri/r87fXPwo4ikm72N7MbAILHZ70A6LqHBKfuVkk2YRRzMT+qJhj88gngAncv9ur+dBrFvCz+OPgQ3lqwiqmL07uwFYmzC4DOQUH3MvCOu98SbiQpjbpbJdmEUczF9KgaM6sHvArc6O4fV3K2lFW/ZjXuHNqV65+bw4atO8KOI5LSzKyHmfUAugP3AGcQOSP3bjBfEpy6WyWZhFHMVfhRNcHyLwCPu/uzVZgxJR3ZtjHHd9qPP7w0v/SFRSQWd0VNfwPWE7nh6y7g7yHmknJQd6ski7gXczE+quZ0oC9wvpnNCqZu8d6HZDZyUAcWrNzIpNkxP7dbRErg7j/by/TDkxjM7Lwwc8reZWYYdw7pwj1vL1F3qyQ0c0/908e5ubmel5cXdoyEMadgAxc8Mo1XrzyK/erXCDuOpDAzm+7uuWHnSFRmNsPdE7bbVW1nxD/fW8qUBat4+uLeZGQUd9m3SOUqb9upJ0CkoS4tGnDu4a24buJs0qGYF0lgqgySwAVHqrtVEpuKuTR1+c8O5LvvdzJOjZNImHQ0lQTU3SqJLqZiLhg3qei8hrFsU+KjWmYGo87oxqgpi1m6ZnPYcUTSlc7MJYk2OXW4/OgDdXerJKRYz8w9b2bV9rwJxn+bEuM2JU4OzKnD1ce255oJsyncVexwfSISAzNrXcq8D+IYR2Kk7lZJVLEWcy8CzwaPqGlF5A7VG2INJfFzTu8DqFcji/v/W+yDNEQkNs8VM2/inhfuPjyOWSRG6m6VRBVTMefuDxM5E/cikdHNL3P3NysjmMRHRoZx55CuPPbhMuYWbAw7jkhKMLMOZnYaUN/MfhE1nQ/oFvIkpu5WSUQVKubMbMSeiUjD1BKYBfQO5kkS2a9+DW46qSNXj5/Jtp27wo4jkgoOAk4EGgAnRU09gItDzCWVQN2tkmiyKrhe3SLvXyhhviSJk7s2480Fq7jjjUXcdFLHsOOIJDV3fwl4ycwOd/ePws4jlWtPd+uQBz/i6INyOKBR7bAjSZqrUDHn7n8sy3Jmdq+7/6Yiv0Piy8y47ZRDGHj3exx7cBOOaNs47EgiqWCmmV0BdCKqe9XdLwwvklSG6O5WDSYsYavqceaOrOLtSyVqUCubv53WmesmzmHj9zvDjiOSCp4A9gOOB94FWgCbQk0klUbdrZIoNGiw/MTRBzXhZx1y+OOk+aUvLCKlaevufwC2uPtjwM+BziFnkkqiu1slUaiYk//xfycczKzlG3huekHYUUSS3Z5T3BvM7BCgPtCqtJXMbKCZLTKzfDMbWcznfc1shpkVmtmQIp+9YWYbzOyVIvNbm9knZrbEzMabWXbFd0v20N2tkgiqupjTRQRJqFZ2Fg+c3ZPbXlvI/JUarkQkBmPNbB/gRmASsAC4fW8rmFkmMAYYBHQEzjKzonclfQWcDzxVzCbuBM4pZv7twGh3bwesBy4q+27I3qi7VcIW6+O8hpYy755Yti/hOWi/uvzx5E78etwMNmzdEXYckaTk7v909/XuPtXd27h7E3d/aM/nZnZeMav1AvLdfam77wCeAQYX2e4yd58D/M+jW9z9bYpcl2dmBhzDjwMWPwacEsu+yY/U3Sphi/XMXHFPe/hhnrs/GuP2JUQndW3GcR335erxs9R9IFI1ripmXnNgedT7gmBeLBoBG9y9sBK3KVH2dLde9+wcPR5R4q6igwYPMrN7geZm9o+o6VGgsJTVJYmMHNSBrTt2cc/bS8KOIpKKirsUpbh5sR5NlXmbZnaJmeWZWd6aNWti/LXp5YIjW1O9WgZ/ff2zsKNImqnombmVQB6wDZgeNU0icgu+pIhqmRnc98vujJ+2nP98tirsOCKppriCqoDIU3X2aEGkzY3FWqCBme0ZW7TEbbr7WHfPdffcnJycGH9tesnMMO47qwdvL1ylG8gkripUzLn77OA2+7bu/ljU9Ly7r6/kjBKyJnVrMGZYd657do6uBxGpXMWdMZsGtAvuPs0GziRyoFxh7u7AO8CeO1/PA16KZZtSvPq1qjH23Fxue20hs5dvCDuOpIlYr5lrZWYTzWyBmS3dM1VKMkkoPQ9oyJX923HZuBl8v0PPbxWpJB8UnRFc1zYcmAwsBCa4+3wzu9XMTgYws0PNrAAYCjxkZj8MDGlm7wHPAv3NrMDM9vSWXA+MMLN8ItfQ/asqdyydtd+3Ln/9RWd+PW46qzdtCzuOpAGLHLBVcGWz94GbgdFEHiJ9QbDNmysnXuXIzc31vLy8sGMkPXdnxITZAIw6vSuRG+RESmZm0909N+wcYTGz6sBpRMaW++Hxie5+a1iZykNtZ2xGT1nMB/lreeri3mRnaVhXKbvytp2x/nXVDG6DN3f/0t1vIXL7u6QgM+Mvp3Zm4dffaTwlkbJ5iciwIoXAlqhJ0sBV/dvRsHY2N+uJOlLFskpfZK+2mVkGsMTMhgMrgCaxx5JEVTM7k4fO6clpD3xIp2b16HlAw7AjiSSyFu4+MOwQEo6MDGPUGd04dcwHjPv4S87ufUDYkSRFxXpm7mqgFnAl0BM4m8iFtZLCDmhUmzuGdOGKJ2fqehCRvfvQzPQs1jRWp3oWD5+by91vLebTL9aFHUdSVEzFnLtPc/fN7l7g7he4+2nu/vGez4Ox6CQFHdNhX04/tCXDn5rJTg2QKVKSPsD04Dmrc8xsrpnNCTuUxFerxrW56/RuDH9qBis3fB92HElBVX1F5pFVvH0J0dX921ErO5PbNUCmSEkGAe2AAURuEjsx+Clppl/7HC7q05pLnshj206NCCCVS7fXSIVlZBh3n9GNyQu+4eXZsY5pKpJ63P1LoAGRAu4koEEwT9LQJX3b0KZxHUY+N4dYRpIQKUrFnMSkQa1sHhjWk5snzWfxqk2lryCSRszsKuBJIjeGNQHGmdlvwk0lYTEzbj+tC0tWb+af730RdhxJIVVdzGkgsjRwSPP6/P6Eg7nsiel8t21n2HFEEslFwGHufpO73wT0Bi4OOZOEqGZ2JmPPzWXse0t5b4mefSuVo6qLuXuqePuSIE7r2YIj2jbitxNms3u3ug9EAgZEXyC1Cx3kpr3mDWpy71nduWb8LD0iUSpFTMWcmb1sZpOKTE+Y2VVmVsPdH62knJIEbjqxE2s2b+fBqZ+HHUUkUTwCfGJmt5jZLcDH6DFaAvRu04gr+7fjksens2V7YdhxJMnFemZuKbAZeDiYvgNWAe2D95JGsrMyuH9YDx79YBnvL1kbdhyR0Ln7KCKPOVwHrAcucPe7w00lieKc3gfQrWUDrlWPhsQo1mKuu7v/0t1fDqazgV7ufgXQoxLySZJpWr8m95zZnavHz6Jg/daw44iEwszqBT8bAsuAccATwJfBPBHMjFtP6cTqTdu47538sONIEou1mMsxs/33vAleNw7e7ohx25KkDj+wEZf2bcPlT87QeEqSrp4Kfk4H8qKmPe9FAKielcmDZ/fk6U+/YsqCVWHHkSQVazF3LfC+mb1jZv8F3gOuM7PawGOxhpPk9aujWtNyn1rcogdMSxpy9xODn63dvU3U1Nrd24SdTxJLk3o1uH9YD0Y+N4f81RriScov1sd5vUZkdPOrg+kgd3/V3bfoupD0ZmbcPqQLeV+u55lPvwo7jkgozOzI4OAWMzvbzEZF92aI7NF9/30YOagDFz8+nY3fa4gnKZ9Y72a9Aqjp7rPdfRZQ08wur5xokuzqVM/iwbN7csfkRcxeviHsOCJheADYamZdgd8BXxK5dk7kfwzNbUm/9jlc+fRMdumGCCmHWLtZL3b3H/4v7e7r0YCYEqVtkzr85dRDuPzJGazbossoJe0UeuS5TYOBe9z9HqBuyJkkgf3+5wezo3A3d05eFHYUSSKxFnMZZvbDAJhmlglkx7hNSTEDD2nKSV2b6WhT0tEmM7sBOBt4NWgjq4WcSRJYtcwMxgzrwStzVjJJz7yWMoq1mJsMTDCz/mZ2DPA08EbssSTV/HZAe3a7c9ebOtqUtHIGsB24yN2/AZoDd4YbSRJdw9rZjD0nl1smzWfeio1hx5EkEGsxdz3wH+DXwBXA20SuCxH5iazMDO49qzsvzVrJ5PnfhB1HJC7c/Rt3H+Xu7wXvv3L3x8POJYmvY7N63Dq4E5c+MZ1vN28PO44kuFjvZt3t7g+4+xB3P83dH3J3DSwmxWpUpzpjhvXg/56fy9I1m8OOI1JlzOz94OcmM/suatpkZt+FnU+Sw4ldmjG4WzMuf3IGO3ftDjuOJLAKFXNmNtfM5pQ0lWH9gWa2yMzyzWxkMZ/3NbMZZlZoZkOKfPaGmW0ws1cqkl3C1a1lA64dcBCXPqHnEUrqcvc+wc+67l4vaqrr7vXCzifJ49oBB1ErO5M/v7Ig7CiSwCp6Zu5E4CQi18e9AQwLpteAiXtbMbgAeAwwCOgInGVmHYss9hVwPj+Ooh7tTuCcCuaWBHBWr5Z0378B1z83h8iNfiKpycx6m1ndqPd1zOywMqwXywHveWa2JJjOi5r/32Cbs4KpSaz7J1UvM8O456zuvLdkLROmLQ87jiSoChVz7v6lu38JHOnuv3P3ucE0Eji+lNV7AfnuvtTddwDPELltP3r7y9x9DvA/55Xd/W1AQ2QnMTPj1sGHsOzbLfzr/S/CjiNSlR4Aoq8p2BrMK1EsB7zBc19vBg4j0tbebGb7RC0yzN27BdPq8u+OhKFejWqMPTeX29/4jBlfrQ87jiSgWG+AqG1mffa8MbMjgdqlrNMciD68KAjmSRqpUS2TB4b15MF3l/Lx0m/DjiNSVcyjTj+7+24gq5R1YjngPR6Y4u7rgnE/pwADY90JCV/bJnW4Y0gXLh83g1XfbQs7jiSYWIu5i4AxZrbMzL4gcjR5QSnrWDHzKr2vzcwuMbM8M8tbs2ZNZW9eKkHLhrUYdXpXrnx6Jt9sVOMkKWmpmV1pZtWC6SpgaSnrxHLAW9q6jwRdrH+IHiNUkkP/g/fl7N77c+kT09m2U/cayo9iLebmAXcA/wZeBF4gci3d3hQALaPetwAqfWREdx/r7rnunpuTk1PZm5dK0rd9DucefgCXPzmdHYW6W0tSzmXAEcAKIm3fYcAlpawTywHv3tYd5u6dgaOCqdhrj3UgnNiu+FlbmjWowR9enKdrjuUHsRZzLxEp3rYRaag2A1tKWWca0M7MWptZNnAmMCnGHJLELj+6LQ1rV+e2V3W3lqQWd1/t7me6exN339fdf1mGa9ViOeAtcV13XxH83ETkWrteJWTWgXACMzPuHNKVuSs28tiHy8KOIwki1mKuRdBQ3eHud+2Z9raCuxcCw4k8PWIhMMHd55vZrWZ2MoCZHWpmBcBQ4CEzm79nfTN7D3gW6G9mBWZW2g0XkuAyMoxRZ3Rl6pK1vDCzIOw4IpXGzNqb2dtmNi9438XMbixltVgOeCcDA8xsn+DGhwHAZDPLMrPGQYZqREYkmFeRfZLw1a6excPn5nLfO5/z4edrw44jCSDWYu5DM+tc3pXc/TV3b+/uB7r7bcG8m9x9UvB6mru3cPfa7t7I3TtFrXuUu+e4e82Jce9DAAAgAElEQVRgmckx7oMkgHo1qvHg2T350ysLWbBSY6pKyngYuAHYCRDctHDm3laI5YDX3dcBfyJSEE4Dbg3mVSdS1M0BZhHp9n24sndW4qdlw1rcc2Y3rnx6FsvXbQ07joTMYulzN7MFQFvgCyLPHzTA3b1L5cSrHLm5uZ6Xlxd2DCmDl2at4K43F/Py8D7Ur6XnkSc7M5vu7rlh5wiLmU1z90PNbKa7dw/mzXL3bmFnKwu1nYnvX+9/wcTpBTz368OplV3ajdKSLMrbdsZ6Zm4Q0I7IqfyT+HEwYZEKGdytOf0PbsLV42eye7cu7pWkt9bMDiS4CSEY4PfrcCNJKrnwyFZ0bFqP303UIOzpLNZns35Z3FRZ4SQ9/d8JB7Nl+y7+8Z8lYUcRidUVwENABzNbAVxN5A5XkUphZtx26iEsX7eVB979POw4EpJYz8yJVLpqmRncN6w7z3y6nHc+0yD1kpzMLAPIdfdjgRygg7v30QGvVLYa1TJ58JyePPbhMrWZaUrFnCSkJnVrcN8vu3PdxNl89a0u7pXkEzztYXjwekswJIhIlWhavyb3D+vBb5+dzdI1m0tfQVKKijlJWLmtGjL8Z225dNx0vt+h0c4lKU0xs9+aWUsza7hnCjuUpKaeBzTkt8cfxMWP57Fp286w40gcqZiThHbeEa1ov28dfv/iXF3cK8noQuBy4F0gL2oSqRJn9dqfww9sxDXjZ+kmsjSiYk4Smpnx1190ZsHK7xj3sS41kqTTkcgzq2cTGd/tXqDTXtcQidFNJ3biu+8LGf3W4rCjSJyomJOEVys7iwfP7sndby1h+pfrw44jUh6PAQcD/yBSyB0czBOpMtlZGdx/dg+en7GC1+dqJJx0oGJOkkKrxrW5Y0gXhj81gzWbtocdR6SsDnL3X7n7O8F0CXBQ2KEk9TWuU50Hz+7J71+cx2ff6Kk6qU7FnCSN/gfvy9CeLRj+1AwKd+0OO45IWcw0s9573pjZYcAHIeaRNNK5RX1uOrEjlzw+nfVbdoQdR6qQijlJKlcd254a1TK5/Y3Pwo4iUhaHEXmG9TIzWwZ8BPQzs7nBc1JFqtQp3ZtzfKd9Gf60DoJTmYo5SSqZGcY9Z3bjjfnf8MqclWHHESnNQKA10C+YWgMnoEcfShxdP7ADGWb89XUdBKcqFXOSdBrUyuaBYT256aX5LFmlcVglcZX0yEM9+lDiKSszg3vP6s5bC1fx3PSCsONIFVAxJ0npkOb1uWFQBy4dN12DY4qIlKJBrWzGnpPLba8tZPbyDWHHkUqmYk6S1tDclhzephG/fXa2BhQWESnFQfvV5S+nduaycdNZvWlb2HGkEqmYk6R200kdWfXddh58d2nYUUREEt7AQ/ZjaG5LLh83gx2FuiEiVaiYk6RWPSuTB87uwb8/+IIP8teGHUdEJOFd3b8d+9TO5uZJ88OOIpVExZwkvab1a3LPmd246plZrNjwfdhxREQSWkaGMer0rkxbto4nP9F9OKlAxZykhCMObMzFR7Xm8nHT2bZzV9hxREQSWt0a1Xj43FxGT1nMtGXrwo4jMVIxJynjkr5taL5PTf74sroORERK07pxbf4+tCtXPDmDlerVSGoq5iRlmBl3DOnKp1+sY/y0r8KOIyKS8I4+qAkX9mnNpU+oVyOZqZiTlFKnehYPnZPLHW8sYk6BxlISESnNpX3b0LpxbW54fq6GeUpSKuYk5bRtUoc/n3IIvx43g3V6uLQkITMbaGaLzCzfzEYW83lfM5thZoVmNqTIZ+eZ2ZJgOi9qfs/gmbD5ZvYPM7N47IskPjPj9tO6sHjVJv71/hdhx5EKUDEnKWlQ56ac2LUpVz0zk127daQpycPMMoExwCCgI3CWmXUssthXwPnAU0XWbQjcDBwG9AJuNrN9go8fAC4B2gXTwCraBUlCNbMzeeicnjw0dSnvLVkTdhwpJxVzkrKuG3AQhbucUVMWhR1FpDx6AfnuvtTddwDPAIOjF3D3Ze4+Byg66uvxwBR3X+fu64EpwEAzawrUc/ePPNKP9jhwSpXviSSVFvvU4t6zunPN+Fl8+e2WsONIOaiYk5SVlZnBvb/szgszVvDm/G/CjiNSVs2B5VHvC4J5sazbPHhdkW1KGundphFX9m/HJY9PZ8v2wrDjSBmpmJOU1rhOdcYM68ENz89l6ZrNYccRKYvirmUr67UCJa1b5m2a2SVmlmdmeWvWqLstHZ3T+wC6tWzAtRNms1uXqSQFFXOS8rrvvw8jBrTnsnHT2bpDR5qS8AqAllHvWwArY1y3IHhd6jbdfay757p7bk5OTplDS+owM249pROrN23jvnfyw44jZaBiTtLCL3vtT9cWDbj+Od16LwlvGtDOzFqbWTZwJjCpjOtOBgaY2T7BjQ8DgMnu/jWwycx6B3exngu8VBXhJTVUz8rkwbN78vSnXzFlwaqw40gpVMxJWjAz/nTKIXyxdjP//mBZ2HFESuTuhcBwIoXZQmCCu883s1vN7GQAMzvUzAqAocBDZjY/WHcd8CciBeE04NZgHsCvgX8C+cDnwOtx3C1JQk3q1eD+YT0Y+dwc8ldvCjuO7IWlw1mK3Nxcz8vLCzuGJIDl67Zy6v0fcP+wnvRq3TDsOCnPzKa7e27YOaRi1HYKwLN5y7n/v5/z4hVHUr9mtbDjpIXytp06MydppWXDWtx1ejd+8/QMVn23Lew4IiIJb2huS/q1z+HKpzVuZ6JSMSdpp1/7HM4+7AAuf3IGOwqLDtMlIiJF/f7nB7OjcDd3Tta4nYlIxZykpSt+1pZ9alXjL68tDDuKiEjCq5aZwZhhPXhlzkomzS7rzdUSLyrmJC1lZBh3nd6N/y5azQszC0pfQUQkzTWsnc3Yc3K5ZdJ85q3YGHYciaJiTtJW/ZrVePCcnvzplYUs/Pq7sOOIiCS8js3qcevgTlz6xHS+3bw97DgSUDEnaa3DfvW4+aSOXDZuOhu/3xl2HBGRhHdil2YM7taMy5+cwc5duu44EaiYk7Q3uFtzfnZQE0aMn6VH14iIlMG1Aw6iVnYmf35lQdhRBBVzIkDkTq2N3+/Uo2tERMogM8O456zuvLdkLROmLQ87TtpTMSdC5E6t+4f14MlPvuSdRavDjiMikvDq1ajG2HNzuf2Nz5jx1fqw46S1UIo5MxtoZovMLN/MRhbzeV8zm2FmhWY2pMhn55nZkmA6L36pJdU1qVeD+37Zg+uenc3ydVvDjiMikvDaNqnD7ad14fJxGog9THEv5swsExgDDAI6AmeZWccii30FnA88VWTdhsDNwGFAL+Dm4GHSIpXi0FYNueJnbbn0iels27kr7DgiIgnv2I77Muyw/dVuhiiMM3O9gHx3X+ruO4BngMHRC7j7MnefAxS9TeZ4YIq7r3P39cAUYGA8Qkv6OP+IVrTbtw6/f2Ee6fDsYhGRWA0/pi1N69fgDy+q3QxDGMVccyD6asmCYF5VrytSJmbGX3/RmXkrNvLkJ1+FHUdEJOGZGX8f2pW5Kzby2IfLwo6TdsIo5qyYeWUt48u8rpldYmZ5Zpa3Zs2aMocTAaiVncWD5/Rk9JTFurBXRKQMalfPYuw5udz3Tj4ffr427DhpJYxirgBoGfW+BVDWB72VeV13H+vuue6em5OTU6Ggkt5aN67N7ad14YonZ7BWI52LiJRq/0a1uPuM7lz59CzdSBZHYRRz04B2ZtbazLKBM4FJZVx3MjDAzPYJbnwYEMwTqRLHdtyXIT1bcN6/P9WzCEVEyqBPu8Zc1q8Nlzwxna07CsOOkxbiXsy5eyEwnEgRthCY4O7zzexWMzsZwMwONbMCYCjwkJnND9ZdB/yJSEE4Dbg1mCdSZa45tj3nHn4A5z8yjRtfnMuGrTvCjiQiktAu6tOajk3rcdbDnzB7+Yaw46Q8S4e7TnJzcz0vLy/sGJLkNmzdwV1vLub1ed/w2wHtOT23JRkZxV3GKXuY2XR3zw07h1SM2k6Jxe7dzsTpBdwxeRHHdMjhuuM7kFO3etixkkJ52049AUKkjBrUyuZPpxzCoxccyoS85Zz6wIfMKdARp4hIcTIyjNMPbcnb1/ajbo1qDBj9Lv98byk7dxUddUxipWJOpJwOaV6fiZcdwdmH7c9Fj+Vxw/NzWb9FXa8iIsWpX7MafzixI89edjjvLl7DwLunMnWxRpmoTCrmRCogI8MYmtuSt0b0o3pWBseNfpcnP/mSXbtT/7IFEZGKaNukLo9f2IuRgw7mxhfn8avH8vjy2y1hx0oJKuZEYlC/ZjVuObkTj194GC/OXMEpYz5gpsalExEplplxXMd9efOavnTfvwGDx3zAnZM/Y8t23fUaCxVzIpWgY7N6TLj0cC44shWXPjGd6yfO4VuNTScVZGYDzWyRmeWb2chiPq9uZuODzz8xs1bB/Gwze8TM5prZbDM7Omqd/wbbnBVMTeK2QyJF1KiWyRU/a8vrVx1Fwfrv6X/Xu7w0a4UeBVZBKuZEKomZ8YseLXjr2n7Urp7FgNFTefyjZep6lXIxs0xgDDAI6AicZWYdiyx2EbDe3dsCo4Hbg/kXA7h7Z+A44C4zi27nh7l7t2BaXZX7IVIWTevX5J4zu3PvL7szdupShj74kcb0rAAVcyKVrF6Natx0UkeevPgwXpnzNSfd+z7Tv9RwiFJmvYB8d1/q7juAZ4DBRZYZDDwWvJ4I9DczI1L8vQ0QFGsbAA0NIwnv0FYNmTS8D6f1bMH5j3zKDc/PVe9GOaiYE6kiHfarx/hLenNpvzZc/uQMrp0wmzWb1DhJqZoDy6PeFwTzil0mGIh9I9AImA0MNrMsM2sN9OSnj0B8JOhi/UNQ/IkkjMwM46xe+/P2iKODG8um8sgHX2gokzJQMSdShcyMwd2a89aIfjSsXY3j7440ToVqnKRkxRVZRfvqS1rm30SKvzzgbuBDYM+V5cOC7tejgumcYn+52SVmlmdmeWvWaPgIib/6tSI3lj1zSW/eWriKn//jPT7IXxt2rISmYk4kDurWqMbvf96R8Zf05s35qzjx3vf59At1vUqxCvjp2bQWwMqSljGzLKA+sM7dC939muCauMFAA2AJgLuvCH5uAp4i0p37P9x9rLvnuntuTk5OJe6WSPm037cu4y46jBHHHcT1z83hsiems3zd1rBjJSQVcyJx1G7fujx18WEMP6YtVz0zk2vGz2L1d9vCjiWJZRrQzsxam1k2cCYwqcgyk4DzgtdDgP+4u5tZLTOrDWBmxwGF7r4g6HZtHMyvBpwIzIvHzojEwswYeMh+vDWiH52a1eOk+95n1JuL+H7HrrCjJRQVcyJxZmac2KUZb43ox771anD83VP1iBv5QXAN3HBgMrAQmODu883sVjM7OVjsX0AjM8sHRgB7hi9pAswws4XA9fzYlVodmGxmc4BZwArg4bjskEglqFEtk9/0b8drVx7FF99upf9d/+Xl2Ss1lEnA0uGL0MOiJZHlr97MH1+ez6rvtnHr4EPo3aZR2JEqTXkfFi2JRW2nJKpPln7LLS8voG6NLG45qRMdm9ULO1KlKm/bqTNzIiFr26QOj1/Yi2uObc+1E2Zz5dMzWaWuVxGREh3WphGv/KYPJ3dtxjn/+oQbX0zvZ2SrmBNJAGbGoM5NmTKiLy0b1mTg3VMZO/Vzdb2KiJQgM8M4u/cBvH1tPzLMOHbUuzz+0bK0HC1AxZxIAqmVncV1x3fg+cuP5IP8bxl0j27JFxHZmwa1srl18CGM+9VhvDb3a068930++vzbsGPFlYo5kQTUunFtHr3gUK47/iB+N3EOVzw5g683fh92LBGRhHVw03o8fXFvruzfjt8+O5srnpxBwfr0GMpExZxIgjIzju8UuSX/wJzanHDPe9z/33x2FKZfF4KISFmYGSd0bspbI/rRbt86nHjv+9z91mK27UztoUxUzIkkuJrZmYwYcBAvXnEkecvWM/DuqUxdrJH5RURKUjM7k6uPbc8rv+nDklWb6X/Xu7w29+uUHcpExZxIkjigUW3+ff6h/N8JB/P7F+dy2RPTWbFBXa8iIiVpsU8txgzrwZ1Du3DPW0v45cOf8Nk334Udq9KpmBNJMsd23Jcp1/Tj4Kb1OPEf73Hff5awvTC1uxBERGJxxIGNefXKPgzqvB/DHv6Em1+ax4atqTOUiYo5kSRUo1omVx3bjknD+zC7YCPHj57KO4tWhx1LRCRhZWVmcO7hrZgyoh+73Dl21LuM+/hLdu1O/q5XFXMiSaxlw1o8fG4uN5/UiT9Oms/Fj+fpQdQiInvRsHY2fz6lM49d2ItJs1Zy0r3v8+kX68KOFRMVcyIp4GcdmvDG1X3p2qI+J9/3Pve8tSTl794SEYlFp2b1GX9pb3599IFc/cxMfvP0TFYm6XXIKuZEUkSNapkMP6YdL/+mDwu//o4Bo6fy9sJVYccSEUlYZsZJXZvx1rX9aN2oFif84z3ufTv5DoZVzImkmBb71OLBc3ryp1MO4bZXF3LRo9P48tstYccSEUlYtbKzGDHgIF4e3of5K7/juNHv8sa8b5JmKBMVcyIpql/7HF6/+ih6ttqHwWM+YNSbi/h+R3IdbYqIxFPLhpGD4b+e2oW73lzEOf/6lCWrNoUdq1Qq5kRSWPWsTC4/ui2vXXkUn6/ZwnGj32Xy/OQ52hQRCUOfdo157aqj6H9wE84Y+zF/fHk+G7/fGXasEqmYE0kDzRrUZMywHvztF124443POP+RaXyxVl2vIiIlqZaZwQVHtmbKNX3ZtnM3/e96l6c//SohhzJRMSeSRvq0a8zrV/XliAMb8Yv7P+DOyZ+xdUdh2LFERBJWozrV+esvOvPoBYcycXoBg8e8T96yxBrKRMWcSJrJzsrg0n4H8vpVfflq3fccN2oqr6fwMwtFRCrDIc3rM/Gyw7n4qDYMf2omVz8zk282bgs7FqBiTiRt7Ve/Bvee1Z07h3Zh1JTFnPvvT/l8zeawY4mIJCwzY3C35rx9bT+a71OTQfdMZcw7+aE/UlHFnEiaO+LAyIW+/drnMOSBD/nb65+xZbu6XkVESlK7ehbXHd+BF684klnLNzBg9FTeWrAqtB4OFXMiQrXMDH51VBsmX92XVd9t49hR7/LKnJXqehUR2YsDGtXm4XNz+dPgQ/jr6ws575Fp5K+Ofw+HijkR+UGTejUYfUY37jmzO/f9J59h//wkKcZYEhEJU9/2ObxxdV/6tmvM6Q99xG2vLuC7bfEbykTFnIj8j16tG/LKb/pwXMd9OWPsx9z26gI2q+s1bsxsoJktMrN8MxtZzOfVzWx88PknZtYqmJ9tZo+Y2Vwzm21mR0et0zOYn29m/zAzi9sOiaSB6B6O774vpP9d7zIhbzm74zCUiYo5ESlWVjDG0uSr+7Juy0763/VfXpq1Ql2vVczMMoExwCCgI3CWmXUssthFwHp3bwuMBm4P5l8M4O6dgeOAu8xsTzv/AHAJ0C6YBlblfoikq5y61bl9SBf+eW4uT3/6Fafe/wEzv1pfpb9TxZyI7FVO3ercdXpXxvyyBw+9u5Qzx35M/mp1vVahXkC+uy919x3AM8DgIssMBh4LXk8E+gdn2joCbwO4+2pgA5BrZk2Beu7+kUeq8ceBU6p+V0TSV9eWDXjusiM474hWXDZuOiMmzOLbzdur5HepmBORMslt1ZCXf9OHn3dpypbtesZrFWoOLI96XxDMK3YZdy8ENgKNgNnAYDPLMrPWQE+gZbB8QSnbFJFKlpFh/KJHC96+9miaN6hJRhVd3ZBVJVsVkZSUmWGce3irsGOkuuJa+6J92yUt82/gYCAP+BL4ECgs4zYjGza7hEh3LPvvv3/ZEovIXtWpnsW1Aw6qsu3rzJyISGIpIHI2bY8WwMqSljGzLKA+sM7dC939Gnfv5u6DgQbAkmD5FqVsEwB3H+vuue6em5OTUyk7JCJVK5Ririru1BIRSRHTgHZm1trMsoEzgUlFlpkEnBe8HgL8x93dzGqZWW0AMzsOKHT3Be7+NbDJzHoH19adC7wUl70RkSoX927WqDu1jiNytDjNzCa5+4KoxX64U8vMziRyp9YZRN2pZWZNgNfN7FB33x3fvRARqRruXmhmw4HJQCbwb3efb2a3AnnuPgn4F/CEmeUD64gUfABNgMlmthtYAZwTtelfA48CNYHXg0lEUkAY18z9cKcWgJntuVMrupgbDNwSvJ4I3FfcnVpmtgHIBT6NT3QRkarn7q8BrxWZd1PU623A0GLWWwYUe2GOu+cBh1RqUBFJCGF0s1bFnVoiIiIiaSmMM3NVcafW//4S3ZElIiIiaSCMM3NVcafW/9AdWSIiIpIOwijmKv1OrXgFFxEREUk0ce9mrcI7tURERETSjqXDQ7PNbA2Ra+xK0xhYW8VxykI5EisDKEdRZc1xgLvrOockpbYzaTOAchSVbDnK1XamRTFXVmaW5+65ypE4ORIhg3Ikbg5JDIny95AIORIhg3KkXw49zktEREQkiamYExEREUliKuZ+amzYAQLK8aNEyADKUVSi5JDEkCh/D4mQIxEygHIUldI5dM2ciIiISBLTmTkRERGRJKZiTkRERCSZuXtKTUSe37oamBc1ryEwhcijv6YA+wTzDfgHkA/MAXpErXNesPwS4Lyo+T2BucE6/yDoqi5jjluIDHY8K5hOiPrshmCbi4Djo+YPDOblAyOj5rcGPgnyjQeyS8jREngHWAjMB66K93eylwxx/T6AGsCnwOwgxx/3ti5QPXifH3zeqqL5ypjjUeCLqO+jW1X/nQbLZgIzgVfC+D40JcaE2s7oDKG3m6XkiPf3obbzf7MkVLsZegNS2RPQF+jBTxuCO/Z8IcBI4Pbg9QnA68F/8N7AJ1H/YJcGP/cJXu/5R/spcHiwzuvAoHLkuAX4bTHLdgz+OKsHfxCfB38omcHrNkB2sEzHYJ0JwJnB6weBX5eQo+meP2CgLrA4+H1x+072kiGu30eQr07wuhqRf1i9S1oXuBx4MHh9JjC+ovnKmONRYEgxy1fZ32mw7AjgKX5slOL6fWhKjAm1ndHbDb3dLCVHvL8PtZ3/u+2EajdTrpvV3acSeQRYtMHAY8Hrx4BTouY/7hEfAw3MrClwPDDF3de5+3oiR2ADg8/quftHHvmv8XjUtsqSoySDgWfcfbu7f0GkGu8VTPnuvtTddwDPAIPNzIBjgInF7FPRHF+7+4zg9SYiR3jN4/md7CVDXL+PYJ82B2+rBZPvZd3o72gi0D/4XeXKV44ce/s+quTv1MxaAD8H/hm839t3WSXfhyQGtZ0/yRB6u1lKjnh/H2o7oyRiu5lyxVwJ9nX3ryHyj4PIM14h8o9iedRyBcG8vc0vKGZ+eQw3szlm9m8z26eCORoBG9y9sDw5zKwV0J3I0Uwo30mRDBDn78PMMs1sFpFunClEjoBKWveH3xd8vjH4XeXNV2oOd9/zfdwWfB+jzax6Bb+P8vw3uRv4HbA7eL+377LKvg9JWGnfdiZCu1lMDlDbGWbbmXDtZroUcyWxYuZ5BeaX1QPAgUA34GvgrnjlMLM6wHPA1e7+3d4WraosxWSI+/fh7rvcvRvQgsgR0MF7WTduOczsECLXT3QADiVy+v/6qsxhZicCq919evTsvawb738vkrjSou1MhHazhBxqO0NqOxO13UyXYm5VcPqU4OfqYH4BkQtM92gBrCxlfoti5peJu68K/hB3Aw8T+QdRkRxriZwuzipLDjOrRqQheNLdnw9mx/U7KS5DWN9H8Ls3AP8lch1FSev+8PuCz+sT6f4pb76y5BgYdKm4u28HHqHi30dZ/06PBE42s2VETuUfQ+SIM7TvQxJO2radidBulpRDbWeobWditpueABfeVvYEtOKnF8/eyU8vWr0jeP1zfnpx5Kf+48WRXxC5MHKf4HXD4LNpwbJ7Lo48oRw5mka9voZIfzlAJ356IeRSIhdBZgWvW/PjhZCdgnWe5acXW15eQgYj0u9/d5H5cftO9pIhrt8HkAM0CF7XBN4DTixpXeAKfnrh6oSK5itjjqZR39fdwN/i8XcaLH80P17IG9fvQ1PiTKjtLK3Niuv3sZccajsToO0kgdrN0BuPyp6Ap4mcdt5JpMK9iEj/9NtEbhl+O+o/mgFjiPT9zwVyo7ZzIZELEvOBC6Lm5wLzgnXuo+TblovL8UTwe+YAk/jpP8jfB9tcRNTdM0TuxlkcfPb7qPltiNx1kx/8EVUvIUcfIqdo5xB1G3s8v5O9ZIjr9wF0IXIr+Zwg7017W5fIbfDPBvM/BdpUNF8Zc/wn+D7mAeP48a6tKvs7LaFRiuv3oSkxJtR2RmcIvd0sJYfazgRoO0mgdlOP8xIRERFJYulyzZyIiIhISlIxJyIiIpLEVMyJiIiIJDEVcyIiIiJJTMWciIiISBJTMSehM7OrzaxW2DlERJKF2k2JpqFJJHTBSNq57r427CwiIslA7aZE05k5iSszq21mr5rZbDObZ2Y3A82Ad8zsnWCZAWb2kZnNMLNng+cSYmbLzOx2M/s0mNoG84cG25ptZlPD2zsRkcqndlNKo2JO4m0gsNLdu7r7IUQevbIS+Jm7/8zMGgM3Ase6ew8gDxgRtf537t6LyMjcdwfzbgKOd/euwMnx2hERkThRuyl7pWJO4m0ucGxwpHiUu28s8nlvoCPwgZnNAs4DDoj6/Omon4cHrz8AHjWzi4k8205EJJWo3ZS9ygo7gKQXd19sZj2JPHvur2b2ZpFFDJji7meVtImir939MjM7jMhDlWeZWTd3/7ays4uIhEHtppRGZ+YkrsysGbDV3ccBfwd6AJuAusEiHwNHRl3XUcvM2kdt4oyonx8Fyxzo7p+4+03AWqBl1e+JiEh8qN2U0ujMnMRbZ+BOM9sN7AR+TeS0/+tm9nVw/cf5wNNmVj1Y50ZgcfC6upl9QuRAZM9R6J1m1o7I0enbwM2Yk0gAACAASURBVOz47IqISFyo3ZS90tAkkjR0K76ISPmo3UwP6mYVERERSWI6MyciIiKSxHRmTkRERCSJqZgTERERSWIq5kRERESSmIo5Efl/9u48vKr63Pv/+05CGGSGMCVkIICIgqCACsGgPc5WxBG0VgalWIfWPp7Wnvax/fn8fOw59tS2akVUqNQBLSpyFKuIqIDIoIAIyDyDzCCDDIH7+WOvtNuYkB3Izto7+byuK5drf9ew79Wrfr33dxQRkSSmZE5EREQkiSmZExEREUliSuZEREREkpiSOREREZEkpmROREREJIkpmRMRERFJYkrmRERERJKYkjkRERGRJKZkTkRERCSJKZkTERERSWJK5kRERESSmJI5ERERkSSmZE5EREQkiaWFHUBVaN68uefm5oYdhkiN8+mnn25394yw45ATo7pTJBwVrTtrRDKXm5vL3Llzww5DpMYxs7VhxyAnTnWnSDgqWneqm1VEREQkiSmZExEREUliSuZEREREkpiSOREREZEkpmROREREJIkpmRMRERFJYkrmRERERJKYkjkRERGRJBbXZM7MLjWzpWa2wszuL+V8jplNMbPPzewDM8uKOpdtZu+a2RIzW2xmuUH598zsMzObb2bTzax9ZcT63uItTF++vTIeJSJSYzw7fTXrdhwIOwyRGi1uyZyZpQJPAJcBnYFBZta5xGW/B8a6e1fgQeDhqHNjgUfc/TSgF7A1KH8SuNnduwEvAr+ujHgPFR3jj+8tq4xHiYjUGDv2HeLpaavCDkOkRotny1wvYIW7r3L3w8A4oH+JazoDU4LjqcXng6Qvzd0nA7j7Pncv/unnQMPguBGwqTKCveT0lmzec5AF63dXxuNERGqEIX3ymLhgE9v3HQo7FJEaK57JXCawPurzhqAs2gLg2uB4ANDAzJoBHYHdZvaamc0zs0eClj6A24BJZrYBuAX4XWUEm5aawuDeuTw7fXVlPE5EpEbIaFCbK7q25rmP14QdikiNFc9kzkop8xKf7wMKzWweUAhsBIqANKBvcL4n0A4YHNxzL3C5u2cBY4A/lPrlZsPNbK6Zzd22bVtMAd/Yqy0fLtvG5j3fxHS9iIjA8L7teP6Ttew7VBR2KCI1UjyTuQ1A26jPWZToEnX3Te5+jbt3B34VlO0J7p0XdNEWAROAs8wsAzjT3WcFj3gZ6F3al7v7KHfv4e49MjIyYgq4YZ1aDOieydiZa2N/SxGRGi63+Sn0bt+ccbPXhR2KSI0Uz2RuDtDBzPLMLB0YCEyMvsDMmptZcQy/BEZH3dskSN4ALgQWA7uARmbWMSi/CFhSmUEP6ZPLy3PWc+CwfmGKiMTqjsJ8np2+msNFx8IORaTGiVsyF7So3QW8QyThesXdF5nZg2Z2VXBZP2CpmS0DWgIPBfceJdLFOsXMFhLpsn06eObtwKtmtoDImLl/r8y4c5qdQo+cJrz62cbKfKyISLV2RmYj2reoz4T5qjtFqlpaPB/u7pOASSXKHog6Hg+ML+PeyUDXUspfB16v3Ei/bVhBHr98bSE398omJaW0oX8iIlLSiMJ8HnjjC647K0t1p0gV0g4QpeiV15R6tVP5YNnW8i8WEREAeuc3o156Gu8t2RJ2KCI1ipK5UpgZwwrytEyJiEgFmBkjCvN58sOVuJdcvEBE4kXJXBmu6NKGFVv3sWTz12GHIiKSNC49oxW79h9m9uqdYYciUmMomStDeloKPzwvl9FqnRMRiVlqijH8/HxGfrgy7FBEagwlc8dxU69s3l28hW17tU2NiFQdM7vUzJaa2Qozu7+U84+a2fzgb5mZ7Q7KL4gqn29mB83s6hL3PmZm++IZ/zVnZbJo09fq2RCpIkrmjqPJKelc0bU1z3+iRYRFpGoEWxc+AVxGZP/qQcF+1f/k7ve6ezd37wY8BrwWlE+NKr8QOAC8G/XsHkDjeL9DnVqpDOmTx1NqnROpEkrmyjG0Tx4vzFrHwSNHww5FRGqGXsCKYAecw8A4oP9xrh8EvFRK+XXA2+5+AP6ZJD4C/LyS4y3Vzedm88GybazfeaAqvk6kRlMyV472LepzRmZDJs7fVP7FIiInLxNYH/V5Q1D2HWaWA+QB75dyeiDfTvLuAia6++ZKivO4GtapxY0922pVAJEqoGQuBsMK8hg9Y7Wm2otIVShttd2yKp+BwPhg15x/PcCsNdCFyA48mFkb4HoiXbLH/3Kz4WY218zmbtu2rUKBlzSsTx6vz9vIjn0adywST0rmYlDQvjnH3JmxYkfYoYhI9bcBaBv1OQsoq2ugZOtbsRuA1939SPC5O9AeWGFma4B6ZraitAe6+yh37+HuPTIyMkq7JGYtGtbh8i6teG6mxh2LxJOSuRiYGUP75PHs9FVhhyIi1d8coIOZ5ZlZOpGEbWLJi8zsVKAJMLOUZ3xrHJ27v+Xurdw9191zgQPu3j4u0Zdwe992PP/JWvYfKqqKrxOpkZTMxejq7pks3LiHFVvjOqNfRGo4dy8iMr7tHWAJ8Iq7LzKzB83sqqhLBwHjvMT4DzPLJdKy92HVRHx87TLqc267poybs778i0XkhCiZi1GdWqnc1CubMTM0mFdE4svdJ7l7R3fPd/eHgrIH3H1i1DW/dffvrEHn7mvcPdPdjx3n+fXjE3npRhTm8+y0VRw5WmZIInISlMxVwA/Oy+F/Fmxi1/7DYYciIpI0umY1Jrf5KVoVQCROlMxVQIsGdbiocytenL0u7FBERJLKiMJ8nvpoJceOaVUAkcqmZK6ChhXkMXbmGg4XqbtARCRWfTs0p1ZqCu9/uTXsUESqHSVzFdS5TUPaNa/PpIVVsu6miEi1YGaMKMxnpLb4Eql0SuZOwLCCPJ6drkWERUQq4rIzWrF17yHmrNkZdigi1YqSuRNwYacW7DtUxJw1u8IORUQkaaSlpjD8/HaM/ECtcyKVScncCUhJMYb0ydUiwiIiFXTd2Vl8vnEPS7/aG3YoItWGkrkTdO1ZWcxevZN1Ow6EHYqISNKoUyuVwb1zeeojtc6JVBYlcyfolNpp3NCzLX/9eE3YoYiIJJUfnJvD+19uZePub8IORaRaiGsyZ2aXmtlSM1thZt9ZqdzMcsxsipl9bmYfmFlW1LlsM3vXzJaY2eJgixos4iEzWxacuyee73A8t56Xy2vzNrD34JHyLxYREQAa1a3FDT3a8sw0DVURqQxxS+bMLBV4ArgM6AwMMrPOJS77PTDW3bsCDwIPR50bCzzi7qcBvYDixYkGE9l3sFNwbly83qE8bRrXpW+HDF7WnoMiIhUytE8er322UTvqiFSCeLbM9QJWuPsqdz9MJOnqX+KazsCU4Hhq8fkg6Utz98kA7r7P3YsHp90BPFi876C7h7oC5bCCPP768RqOalVzEZGYtWpUh0tPb8VzM9eEHYpI0otnMpcJRDdZbQjKoi0Arg2OBwANzKwZ0BHYbWavmdk8M3skaOkDyAduNLO5Zva2mXWI4zuUq1vbxrRsWId3F30VZhgiIklneGE7/jZzLQcOF4UdikhSi2cyZ6WUlWy+ug8oNLN5QCGwESgC0oC+wfmeQDsi3asAtYGD7t4DeBoYXeqXmw0PEr6527ZtO8lXOb7iRYRFRCR2+Rn16ZHbhFc0VEXkpMQzmdtAZGxbsSxgU/QF7r7J3a9x9+7Ar4KyPcG984Iu2iJgAnBW1HNfDY5fB7qW9uXuPsrde7h7j4yMjMp6p1Jd3Lklm/ccZMH63XH9HhGR6mZEYT5PT1vNkaPa71rkRMUzmZsDdDCzPDNLBwYCE6MvMLPmZlYcwy/5VyvbHKCJmRVnYRcCi4PjCcFniLTmLYtT/DFLS00JFhFW65yISEV0z25C26Z1efPzTeVfLCKlilsyF7So3QW8AywBXnH3RWb2oJldFVzWD1hqZsuAlsBDwb1HiXSxTjGzhUS6bJ8O7vkdcG1Q/jBwW7zeoSJu6NmWj5ZvY/MerZskIlIRIwrzeerDVdrvWuQEpcXz4e4+CZhUouyBqOPxwPgy7p1MKV2o7r4buKJyIz15DevUYkD3TJ77eC33X9Yp7HBERJJGYccM/vMfS/lg6TYu6NQi7HBEko52gKhEg3vn8vKcdZqZJSJSAWbGiMJ2PPmBtvgSORFK5ipRTrNT6JnblFc/3RB2KCKSxGLYPedRM5sf/C0zs91B+QVR5fPN7KCZXR2ceyF45hdmNtrMalX1ex3PFV1as/nrb/h07a6wQxFJOkrmKtmwgjxGz1jDMS0iLCInIJbdc9z9Xnfv5u7dgMeA14LyqVHlFwIHgHeD214AOgFdgLokyHjjYmmpKdzetx0jP1TrnEhFKZmrZL3ymnJK7VSmLg11YwoRSV6x7J4TbRDwUinl1wFvF++e4+6TPADMJrJcVEK5/uy2zFu3i+Vb9oYdikhSUTJXycxMiwiLyMmIZfccAMwsB8gD3i/l9EBKSfKC7tVbgH+cdKSVrG56Kreel8tTH60KOxSRpKJkLg6u6NKGldv2sXjT12GHIiLJJ5bdc4oNBMYHyzn96wFmrYl0p75Tyj1/AT5y92mlfnkV7p5TmlvOy2Hy4i1s2q1lnkRipWQuDtLTUvjhebmMnqHWORGpsHJ3z4lSausbcAPwursfiS40s98AGcDPyvryqtw9pzSN66Vz/dlZ6t0QqQAlc3FyU69s3l30FVv3Hgw7FBFJLuXungNgZqcCTYCZpTzjO+PozOw24BJgkLsn9N5Zw/rmMf7TDew+cDjsUESSgpK5OGlySjpXntmG5z9ZF3YoIpJEYtw9ByIJ2zgvsW2CmeUSadn7sMSjRxLZaWdmsGzJAySo1o3qclHnlvxt5tqwQxFJCnHdAaKmG9onj4GjZvLjfvnUqZUadjgikiTK2z0n+PzbMu5dQykTJtw9qer7EYXtGDjqE27r24666ao/RY5HLXNx1L5Ffc7IbMQb8zeGHYqISFJp36IB3bOb8PdP15d/sUgNp2QuzoqXKdEG0iIiFTOiMJ9RH62i6GhCD/ETCZ2SuTgraN8cw5i+YnvYoYiIJJWzc5rQplFd3lq4OexQRBKakrk4MzOGFuRqmr2IyAm4o18+Iz9cpd4NkeNQMlcF+nfL5IuNe1ixdV/YoYiIJJV+p2bg7ny4rOoXMBZJFkrmqkCdWqncdE4OY7SIsIhIhZgZPypsx8gPV4YdikjCUjJXRW45N4c3P9/Mrv1aBFNEpCKu7NqG9Tu/Yd66XWGHIpKQlMxVkYwGtbmoc0tenK1FhEVEKqJWagq3981T65xIGZTMVaGhffL428y1HC7SNHsRkYq4oWdb5q7ZpbHHIqVQMleFOrdpSLuMU5ikafYiIhVSLz2NH56Xy6iP1DonUpKSuSqmRYRFRE7MD8/L4Z1FW/hqz8GwQxFJKErmqtgFp7Zg/6Ei5qzRQF4RkYpocko615yVyWitDCDyLXFN5szsUjNbamYrzOz+Us7nmNkUM/vczD4ws6yoc9lm9q6ZLTGzxWaWW+Lex8ws6QZPpKQYQ/rk8uz0VWGHIiKSdG7r245X5q5nz4EjYYcikjDilsyZWSrwBHAZ0BkYZGadS1z2e2Csu3cFHgQejjo3FnjE3U8DegFbo57dA2gcr9jj7Zqzspi9eifrdhwIOxQRkaSS2bguF3ZqwfOz1oYdikjCiGfLXC9ghbuvcvfDwDigf4lrOgNTguOpxeeDpC/N3ScDuPs+dz8QnEsFHgF+HsfY4+qU2mnc0LMtYz5WV4GISEWNKMxnzIw1HDxyNOxQRBJCPJO5TGB91OcNQVm0BcC1wfEAoIGZNQM6ArvN7DUzm2dmjwRJHMBdwER3T+opobeel8trn23k64PqKhARqYiOLRtwZlYjxn+6IexQRBJCPJM5K6Ws5BTO+4BCM5sHFAIbgSIgDegbnO8JtAMGm1kb4HrgsXK/3Gy4mc01s7nbtiXenn5tGtfl/I4ZvDJnffkXi4jIt9zRL59RH62i6KjW7RSJZzK3AWgb9TkL2BR9gbtvcvdr3L078KugbE9w77ygi7YImACcBXQH2gMrzGwNUM/MVpT25e4+yt17uHuPjIyMSn61yjGsII8xM9aoMhIRqaAeuU1p0aA2b3/xVdihiIQunsncHKCDmeWZWTowEJgYfYGZNTez4hh+CYyOureJmRVnYRcCi939LXdv5e657p4LHHD39nF8h7jq1rYxrRrV4d3FW8IORUQk6YwozGfkhyu1bqfUeHFL5oIWtbuAd4AlwCvuvsjMHjSzq4LL+gFLzWwZ0BJ4KLj3KJEu1ilmtpBIl+3T8Yo1TMWLCIuIFIthWadHzWx+8LfMzHYH5RdElc83s4NmdnVwLs/MZpnZcjN7OfiRndQu7NSCI0ePMW359rBDEQlVWjwf7u6TgEklyh6IOh4PjC/j3slA13KeX78SwgzVxZ1b8tBbS5i/fjfd2ibtaisiUkmilnW6iMiQkzlmNtHdFxdf4+73Rl1/N5EhKLj7VKBbUN4UWAG8G1z6n8Cj7j7OzEYCw4An4/9G8ZOSYvzo/Ejr3PkdE3M4jUhV0A4QIUtLTQkWEVbrnIgAsS3rFG0Q8FIp5dcBb7v7ATMzIsNVin88PwdcXYkxh+aqbm1Ys30/C9bvDjsUkdAomUsAN/Rsy0fLtrFp9zdhhyIi4YtlWScgsosOkAe8X8rpgfwryWsG7A6Gvxz3mcmmVmoKt/Vtx8gPV4YdikholMwlgIZ1anHNWZk8N3NN2KGISPhiWdap2EBgfDDO+F8PMGsNdCEyZrlCz0z0ZZ1KM7BXW2av3smqbUm3w6NIpVAylyCG9M7jlTnr2X+oqPyLRaQ6K3dZpyjRrW/RbgBed/fiVcm3A43NrHicdJnPTIZlnUqql57Gzefm8PQ07XktNZOSuQSR3awevfKa8upnWtFcpIYrd1knADM7FWgCzCzlGd8aR+eRtTumEhlHB3Ar8EYlxx2qwb1zmbTwK7Z+fTDsUESqnJK5BDKsoB1jZqzh2DGtmSRSU8W4rBNEErZxXmKRNTPLJdKy92GJR/8C+Fmw0Hoz4Nn4vEE4mp6SzoDumTw7Q5PJpOaJ69IkUjE9c5tQv3Ya73+5lX/r3DLscEQkJOUt6xR8/m0Z966hlMkN7r6KyEzZamtYQR7ff3w6d17QnoZ1aoUdjkiVUctcAjEzhhXkMVq/LEVEKqxt03r065jBC5+sCzsUkSqlZC7BXN6lNau27Wfxpq/DDkVEJOmM6JfP6BmrOXjkaPkXi1QTSuYSTHpaCrecl6PWORGRE9CpVUPOaNOQ1z7bGHYoIlVGyVwCuvmcbCYv3sLWvZqVJSJSUSMK8xn10UqOajKZ1BBK5hJQ43rpXNm1Nc9r3IeISIX1ymtK01PS+ccXX4UdikiVUDKXoIYW5PHirLUa9yEiUkFmxojCfEZ+uJISK7eIVEtK5hJUfkZ9umQ24o35GvchIlJR/3ZaS745cpSPV+4IOxSRuFMyl8CGFuTx7PTV+mUpIlJBKSnGj85vx5MfrAw7FJG4UzKXwAraN8cwpq/YHnYoIiJJp3+3TFZu28fCDXvCDkUkrpTMJTAzY2hBLs9O1zIlIiIVlZ6WwrCCPEZ+pNY5qd6UzCW4/t0y+WLjHlZs3Rt2KCIiSWdgr2xmrtzBmu37ww5FJG6UzCW4OrVSuemcHEbPWBN2KCIiSad+7TRuPiebUdNWhR2KSNwomUsCt5ybw5sLNrFr/+GwQxERSTq39s7lrc83ayF2qbaUzCWBjAa1ufj0Vrw4W4sIi4hUVPP6tbnqzDb8VT0cUk0pmUsSQ/vkMXbmGg4XHQs7FBGRpDP8/Ha8NHsdew8eCTsUkUoX12TOzC41s6VmtsLM7i/lfI6ZTTGzz83sAzPLijqXbWbvmtkSM1tsZrlB+QvBM78ws9FmViue75AoOrdpSH5Gfd5auCnsUEREkk7bpvXo2yGDF2eph0Oqn7glc2aWCjwBXAZ0BgaZWecSl/0eGOvuXYEHgYejzo0FHnH304BewNag/AWgE9AFqAvcFq93SDTDtIiwiMgJ+1FhO0bPWM2hIm2TKNVLPFvmegEr3H2Vux8GxgH9S1zTGZgSHE8tPh8kfWnuPhnA3fe5+4HgeJIHgNlAFjXEBae24MCho8xevTPsUEREks7pbRrRqVVDXv9M2yRK9RLPZC4TWB/1eUNQFm0BcG1wPABoYGbNgI7AbjN7zczmmdkjQUvfPwXdq7cA/4hL9AkoJcUY0keLCIuInKgRhfmM+mgVR4+ph0Oqj3gmc1ZKWcl/e+4DCs1sHlAIbASKgDSgb3C+J9AOGFzi3r8AH7n7tFK/3Gy4mc01s7nbtm074ZdINNeencWcNTtZu0MLYIqIVNS57ZrSoG4tJi/+KuxQRCpNPJO5DUDbqM9ZwLdG77v7Jne/xt27A78KyvYE984LumiLgAnAWcX3mdlvgAzgZ2V9ubuPcvce7t4jIyOjst4pdPXS07ixZzZjNMVepNqKYfLYo2Y2P/hbZma7o86VNXnse2b2WXDPdDNrX3VvlDjMjDsK83nyg5UafyzVRjyTuTlABzPLM7N0YCAwMfoCM2tuZsUx/BIYHXVvEzMrzsIuBBYH99wGXAIMcvcauU7Hrb1zeH3eRr7WFHuRaieWyWPufq+7d3P3bsBjwGtRp8uaPPYkcHNwz4vAr+P7Jonr4s4t2XuoiJmrdoQdikiliFsyF7So3QW8AywBXnH3RWb2oJldFVzWD1hqZsuAlsBDwb1HiXSxTjGzhUS6bJ8O7hkZXDsz+IX5QLzeIVG1blSXwo4ZvDx7ffkXi0iyiWXyWLRBwEtw/MljRIa5NAyOG1Gip6QmSUkxfnR+O0Z+qC2+pHpIi+fD3X0SMKlE2QNRx+OB8WXcOxnoWkp5XGNOFsMK8vjxC58xpE8uaala+1mkGilt8tg5pV1oZjlAHvB+UPTPyWNB+XvA/cEP5NuASWb2DfA1cG4ZzxwODAfIzs4+6ZdJVFd3z+QPk5exaNMeTm/TKOxwRE6KsoAkdWbbxrRuVId3Fm0JOxQRqVyxTB4rNhAYHyRrcPzJY/cCl7t7FjAG+ENpD6yu441Lqp2WyrCCPLXOSbUQUzJnET8o7tIMBtj2im9oUp5hBXmMnqFlSkSqmXInj0UZSNDFGnXvdyaPBeOPz3T3WcF1LwO9Kzfs5DOoVzbTl29j3Y4D5V8sksBibZn7C3AekbEZAHuJDNCVEF18eiu2fH2Q+et3l3+xiCSLciePAZjZqUATYGaJe0ubPLYLaGRmHYPyi4iMZa7RGtSpxaBe2Tw9Ta1zktxiTebOcfc7gYMA7r4LSI9bVBKT1BRjcG8tIiySyII9qP8tOK5rZg2Od32Mk8cg8uN6nEetr1HW5LHgmbcDr5rZAiILrv975b1l8hrSJ4+JCzaxfd+hsEMROWGxTiY4EkyXd4DgV1+NXBYk0dzYsy2PT13Bpt3f0KZx3bDDEZEoZnY7kckETYF8Il2mI4HvHe++8iaPBZ9/W8a9ZU0eex14Pfboa4aMBrW5smtr/jpjDfddcmrY4YickFhb5v5MpBJoYWYPAdOB/xu3qCRmDerU4pruWTw3c03YoYjId90J9CEyexR3Xw60CDUi+Y7h57fjhVlr2XeoKOxQRE5ITMmcu78A/Bx4GNgMXO3uf49nYBK7wb1zeWXOevarIhJJNIeCteIAMLM0yp6ZKiHJaXYKfdo356VZ68IOReSExDqbNR9Y7e5PAF8AF5lZ47hGJjHLblaPXnlNefWzDWGHIiLf9qGZ/QdQ18wuAv4O/E/IMUkpRhTm8+z01Rwu0ggiST6xdrO+ChwN9vJ7hshilC/GLSqpsGEF7RgzYw3HjulHv0gCuR/YBiwEfkRkHFyN3UYrkZ2R2YgOLeszYf7GsEMRqbBYk7ljwWyoa4A/ufu9QOv4hSUV1TO3CfVrp/H+l1vLv1hEqoS7H3P3p939ene/LjjWL64ENaIwn5EfrtSPYkk6sSZzR8xsEPBD4M2grFZ8QpITYWYMK8jTMiUiCcTMOpjZeDNbbGariv/CjktK1zu/GfVrpzF5iXbWkeQSazI3hMiiwQ+5+2ozywOej19YciIu79Ka1dv3s2jTnrBDEZGIMcCTQBFwATAW+FuoEUmZzOyfrXNqQJVkEuts1sXufo+7vxR8Xu3uv4tvaFJR6Wkp3HJeDqOnrwk7FBGJqOvuUwBz97XB2nAXhhyTHMclp7di94EjzF69M+xQRGIW62zWK81snpntNLOvzWyvmX0d7+Ck4m4+J5vJi79i696DYYciInDQzFKA5WZ2l5kNQOvMJbTUFGP4+e148sOVYYciErNYu1n/CNwKNHP3hu7ewN0bxjEuOUGN66Xz/TPb8PzMtWGHIiLwU6AecA9wNvADImOPJYEN6J7J4k1fs2Sz2iwkOcSazK0HvtAsrOQwtCCPF2ev4+CRo2GHIlLTOZExchOBHkBH4OlQI5Jy1amVypA+eTyl1jlJErHuzfpzYJKZfQj8czdid/9DXKKSk5KfUZ8umY2YMG8jA3tlhx2OSE32ApEN7Rei/ayTys3nZnP+f01l/c4DtG1aL+xwRI4r1pa5h4ADQB2gQdSfJKhhBe0YPWO1ZmSJhGubu08MJo2tLf4LOygpX8M6tRjYM5tnpmklGUl8sbbMNXX3i+MaiVSqPu2bkWLGtOXbOb9jRtjhiNRUvzGzZ4ApfLtX47XwQpJYDe2Ty0WPfsQ93+tAs/q1ww5HpEyxtsy9Z2ZK5pKImTG0jxYRFgnZEKAbcCnw/eDvylAjkpi1aFiHy7u05rmP14QdishxlZvMmZkRGTP3DzP7RkuTJI+rurVh0aavWbF1b9ihiNRUZ7p7D3e/1d2HBH9Dww5KYjf8/HY8P2sd+w8VhR2KSJnKTeaCGazz3T3F3etqaZLkUadWKjefk82zWkRYJCyfmFnnsIOQE5fX/BTObdeUUf3zagAAIABJREFUcXPWhx2KSJli7WadaWY9K/pwM7vUzJaa2Qozu7+U8zlmNsXMPjezD8wsK+pctpm9a2ZLgn0Nc4PyPDObZWbLzexlM0uvaFw1yQ/OzeGtzzexc//hsEMRqYkKgPlBPfi5mS00s8/DDkoqZkRhPs9MW8XhIk1IlsQUazJ3AZFfmCtjrZDMLBV4ArgM6AwMKuUX6u+Bse7eFXgQeDjq3FjgEXc/DegFbA3K/xN41N07ALuAYTG+Q42U0aA2l5zeihdnaQKdSAguBToAF/Ov8XLfDzUiqbCuWY1pl3EKExdsCjsUkVLFmsxdBrQjsqdgrBVSL2CFu69y98PAOKB/iWs6E5nlBTC1+HyQ9KW5+2QAd9/n7geC8XsXAuODe54Dro7xHWqsYX3zGDtzrX5VilSx6OVIKrI0SQy9Go+a2fzgb5mZ7Y46V1avhpnZQ8H1S8zsnsp81+puRGE+T324kmPHtNyTJJ6YkrkTrJAyiewcUWxDUBZtAXBtcDwAaGBmzYiskr7bzF4L9oR9JGjpawbsdvei4zxTSujUqiEdWtbnrYX6VSmS6GLp1XD3e929m7t3Ax4Dopc6KatXYzDQFugUnBsX1xepZgraNyc9LYX3v9xa/sUiVSzWlrkTYaWUlfxJcx9QaGbzgEJgI1BEZP27vsH5nkRaBQfH+MzIl5sNN7O5ZjZ327ZtJ/QC1cmwgsgyJVpEWCThxdKrEW0Q8BKU3asRXHcH8KC7HwvOKSupADPjjn75PKktviQBxTOZ20DkV2CxLOBbTUPuvsndr3H37sCvgrI9wb3zgsqsCJgAnAVsBxqbWVpZz4x69qhgSYAeGRlaNLdfxxYcOHSU2at3hh2KiBxfLL0aQGQSGZAHvB8UldWrAZAP3Bj8yH3bzDqU8Uz9EC7DZWe0Zvu+Q8xZo3pUEks8k7k5QIdg9mk6MJDIZtP/ZGbNzaw4hl8Co6PubWJmxVnYhcDiYJmUqcB1QfmtwBtxfIdqIyXFGNInV4sIiyS+mHsgiNSr4939aPC5rF4NgNrAQXfvATzNv+rbb3+RfgiXKTXFuL1vO0Z+oNY5SSxxS+aCFrW7gHeAJcAr7r7IzB40s6uCy/oBS81sGdCSyB6wBBXTfcAUM1tIpHJ7OrjnF8DPzGwFkTF0z8brHaqba8/OYs6anazdsT/sUESkbOX2akQZSNDFGnVvab0axedeDY5fB7pWWsQ1yHVnZ/H5xj0s/UqLsUviiGfLHO4+yd07unu+uxcnag+4+8TgeLy7dwiuuc3do/cunOzuXd29i7sPDsaOEFRSvdy9vbtfH32PHF+99DRu7JnNmBlrwg5FRMpWbq8GgJmdCjQBZpa49zu9GsHxhOAzRMYoL4tD7NVenVqpDO6dy1MaOycJJK7JnCSeW3vn8Pq8jXx98EjYoYhIKWLs1YDIxIdxHjWrqZxejd8B1wblDwO3xf9tqqcfnJvD+0u3smHXgfIvFqkCaeVfItVJ60Z1KeyYwcuz13P7+e3CDkdESuHuk4BJJcoeKPH5t2XcO5lSulDdfTdwReVFWXM1qluLG3u05Zlpq/ntVaeHHY6IWuZqomEFefz14zUUHdUiwiIiJ2JoQR6vz9vILm2VKAlAyVwNdGbbxrRuVId3Fm0JOxQRkaTUsmEdLj29Fc/NXBN2KCJK5mqqyCLCq8IOQ0QkaQ0vbMffZq7lwOGi8i8WiSMlczXUxae3YuveQ8xbtyvsUEREklJ+Rn165jbl5Tnry79YJI6UzNVQqSnG4N5aRFhE5GSM6JfPM9NWc0RjkCVESuZqsBt7tmXa8u1s3P1N2KGIiCSlbm0b07ZpXd78vKx1nUXiT8lcDdagTi2uPSuLsR+vCTsUEZGkdUe/9oz8YBVRS/6JVCklczXckD65vDJ3PfsPaQCviMiJOL9Dc1JTjKlLt4YditRQSuZquLZN63FOXjPGf7oh7FBERJKSmfGjwnaM/EArBEg4lMwJw/rmMWbGao4dUxeBiMiJuKJLazZ//Q2frt0ZdihSAymZE3rkNKFh3VpM+VJdBCIiJyItNYXhfdvx5ykrOKofxlLFlMwJZqZFhEVETtL1PdpyqOgoPxw9i617D4YdjtQgSuYEgMu7tGbN9gMs2rQn7FBERJJSnVqpvHDbufTIacqVf57OtOXbwg5JagglcwJArdQUftg7R4sIi4ichNQU496LOvLHgd34979/ziPvfEmRFhSWOFMyJ/90U69s3lu8ha1fq3tARORk9M5vzpv3FLBw49cMHPUJm7Q4u8SRkjn5p8b10rmqWxv+9snasEMREUl6zevX5q+De/K901py1eMzeG/xlrBDkmpKyZx8y5A+ebw4ax0HjxwNOxQRkaSXkmLc0S+fp245i99MXMT/eXMxh4vU7SqVS8mcfEt+Rn26ZjViwryNYYciIlJtnJ3TlLfuKWDdzgNcN/Jj1u04EHZIUo0omZPvGFbQjmenr9ZaSSIilahxvXRG3XI2A7pnMuAvM3jr881hhyTVhJI5+Y4+7ZvRpnFdBj39Cet36tejSFUzs0vNbKmZrTCz+0s5/6iZzQ/+lpnZ7qhz2Wb2rpktMbPFZpZb4t7HzGxf/N9CSmNmDOmTx5ghPfmvd77kV68v1LAWOWlxTeZiqJByzGyKmX1uZh+YWVbUuaNRldXEqPLvmdlnQfl0M2sfz3eoicyM0YN78r1OLej/xAxe/XQD7mqlE6kKZpYKPAFcBnQGBplZ5+hr3P1ed+/m7t2Ax4DXok6PBR5x99OAXsA/t3Yxsx5A4zi/gsSga1Zj3ry7gD3fHOHqJ2awYqvyazlxcUvmYqmQgN8DY929K/Ag8HDUuW+KKyt3vyqq/Eng5qASexH4dbzeoSZLTTF+VJjP88POYdRHq/jxC5+xa//hsMMSqQl6ASvcfZW7HwbGAf2Pc/0g4CWAoI5Nc/fJAO6+z90PBOdSgUeAn8czeIldgzq1eGxQd27tncsNT83ktc82hB2SJKl4tszFUiF1BqYEx1NLOV8aBxoGx42ATZUQq5Shc5uGvHFXHzIb1+XSP33EB0u1f6tInGUC66M+bwjKvsPMcoA84P2gqCOw28xeM7N5ZvZIkMQB3AVMdPfjDtQys+FmNtfM5m7bph0M4s3MGNQrmxdvP4cnpq7gf72ygP2HisIOS5JMPJO5WCqkBcC1wfEAoIGZNQs+1wkqlE/M7Oqoe24DJpnZBuAW4HeVH7pEq1MrlV9f2ZlHb+jGf7y2kAfe+IJvDmuMh0icWCllZY1zGAiMd/fifyHTgL7AfUBPoB0w2MzaANcT6ZI9Lncf5e493L1HRkZGhYOXE9OpVUP+5+4CUgyuenw6SzZ/HXZIkkTimczFUiHdBxSa2TygENgIFP8kyXb3HsBNwB/NLD8ovxe43N2zgDHAH0r9cv26rHS92zfn7Z+cz55vjnDlY9NYuEH7uIrEwQagbdTnLMrugRhI0MUade+8oEekCJgAnAV0B9oDK8xsDVDPzFZUduBycuqlp/HI9Wdy5wXtufmZWbw4a53GK0tM4pnMlVshufsmd7/G3bsDvwrK9hSfC/65CvgA6G5mGcCZ7j4reMTLQO/Svly/LuOjUb1a/Glgd37ybx0ZPGY2j01Zrn0HRSrXHKCDmeWZWTqRhG1iyYvM7FSgCTCzxL1NgroS4EJgsbu/5e6t3D3X3XOBA+6uyWMJ6pqzsvj7iPMYO3MNd700j68PHgk7JElw8Uzmyq2QzKy5mRXH8EtgdFDexMxqF18D9AEWA7uARmbWMbjnImBJHN9BynDVmW14854CPlm9gxuemsnaHfvDDkmkWgha1O4C3iFSv73i7ovM7EEzi54MNggY51FNN0F3633AFDNbSKSH5Omqi14qS35GfSbc2Yem9dK58s/T+XzD7vJvkhrL4tmEa2aXA38EUoHR7v6QmT0IzHX3iWZ2HZEZrA58BNzp7ofMrDfwFHCMSML5R3d/NnjmACIzX48RSe6GBq13ZerRo4fPnTs3Pi9Zwx075vz14zU8PnUFP7/kVG7s2Raz0nrYpSYys0+D4RKShFR3JoZJCzfzvyd8wY8vaM/QPrmqY2uAitadcU3mEoUqpPhbtmUvPx03nzaN6/K7a7vQvH7tsEOSBKBkLrmp7kwc63Yc4O6XPiOjQR1+f31XGtdLDzskiaOK1p3aAUIqRceWDZhwZx/at6jP5X+axpQlW8IOSUSk2shuVo+/j+hNbrN6XPHn6cxdszPskCSBKJmTSpOelsL9l3Xi8ZvO4jcTF/HL1xZqvSQRkUqSnpbCr6/szIP9T2fE85/xxNQVHNMe2oKSOYmDXnlNefsnfTlcdIwr/jyNz9btCjskEZFq43untWTiXX34YOlWbh0zm217D4UdkoRMyZzERYM6tfjvG87kF5d2YvjYT/nD5GUc0RImIiKVok3jurx0+7mcmdWYKx+bxscrtocdkoRIyZzE1WVdWjPpngIWrN/NdU9+zMpt2kxaRKQypKWmcN8lp/L768/kpy/P5w/vLtW6nzWUkjmJuxYN6/DXIT257uwsrh85k799slarmouIVJK+HTJ4854CPl23i5uemcVXew6GHZJUMSVzUiXMjFvOy+WVH53H3+euZ8hf57B1ryocEZHK0KJBHcYOPYfzOzTnysemM/XLrWGHJFVIyZxUqfYt6vPqHb3pmtmIy/80nX988VXYIYmIVAupKcZdF3bgLzefxX+8vpCHJy3RWOUaQsmcVLlaqSn87OJTeeqWs3n47SX8+98XsFd7D4qIVIpeeU15656+LN+6j+tHzmT9zgNhhyRxpmROQnN2ThMm3dOXtFTj8j9PY44WwRQRqRRNT0nn2Vt7cGXX1lz9xAz+8cXmsEOSOFIyJ6E6pXYaD1/TlQeuPJ0fv/AZ//mPLzlcpG4BEZGTZWbc1rcdzw7uyUOTlvCbN77g4JGjYYclcaBkThLCRZ1b8vZP+rJ8y14G/GUGy7fsDTskEZFqoVvbxrx5d1+27j3EtU9+zOrt+8MOSSqZkjlJGM3r1+bpH/bgB+fmcOOoTxgzY7W2qhERqQSN6tbiLzefxcCebbn2yY95Y/7GsEOSSqRkThKKmTGoVzav3dGbiQs2ceuY2VozSUSkEhQvEfW3Yb3403vL+cX4z/nmsLpdqwMlc5KQcpufwt9/dB49c5ty5WPT+J8Fm8IOSUSkWji9TSMm3l3AoaKjXPX4dJZpWEvSUzInCSstNYV7vteBZ2/tyaPvLeOn4+ax5xstYSIicrLq107j0Ru7cfv57Rg46hNembNeO/MkMSVzkvDObNuYt+7uS4M6tbj8T9P4eKU2lBYROVlmxg092vLy8HN5dvpqfvryfPYdKgo7LDkBSuYkKdRNT+X/XH0G//+AM7j35fk89NZiDhVprIeIyMnq0LIBE+7sQ730VL7/2HS+2Lgn7JCkgpTMSVK54NQWvP2T81m/8xv6Pz6DJZu/DjskkUpnZpea2VIzW2Fm95dy/lEzmx/8LTOz3VHnss3sXTNbYmaLzSw3KH8heOYXZjbazGpV3RtJoqubnsrD13Tl3os6cuvo2Tz38Rp1uyYRJXOSdJqeks6TPziL2/q24+ZnZjHqo5VawkSqDTNLBZ4ALgM6A4PMrHP0Ne5+r7t3c/duwGPAa1GnxwKPuPtpQC+geMf1F4BOQBegLnBbXF9EktJVZ7bh1Tt68/dP1zPi+U/Zc0DjlJOBkjlJSmbGdWdn8cadfZi8eAs3PfMJG3Zp/0GpFnoBK9x9lbsfBsYB/Y9z/SDgJYAg6Utz98kA7r7P3Q8Ex5M8AMwGsuL5EpK8cpufwqt39KZ1o7pc8dg0Plu3K+yQpBxK5iSptW1aj3HDz6OwYwv6Pz6D1+dtUNeAJLtMYH3U5w1B2XeYWQ6QB7wfFHUEdpvZa2Y2z8weCVr6ou+pBdwC/KOMZw43s7lmNnfbtm0n+SqSrGqnpfLbq07nf1/ZmeFj5/LUh+oBSWRxTeZiGPeRY2ZTzOxzM/vAzLKizh2NGhMyMarczOyhYJzIEjO7J57vIIkvNcW4o18+zw3txV+mruSuF+ex+8DhsMMSOVFWSllZ/xUdCIx39+LZQGlAX+A+oCfQDhhc4p6/AB+5+7TSHujuo9y9h7v3yMjIqGjsUs1ccnorJtzZh3cWfcXQ5+awY9+hsEOSUsQtmYtl3Afwe2Csu3cFHgQejjr3TfGYEHe/Kqp8MNAW6BSMCRkXr3eQ5HJGZiP+5+4CWjasw2V/msa05WpVkKS0gUgdVywLKGvV7IEEXaxR984LumiLgAnAWcUnzew3QAbws0qNWKq1rCb1ePlH59GpVUOufGw6n6zaEXZIUkI8W+ZiGffRGZgSHE8t5Xxp7gAedPdjAO6+tZzrpQapUyuVB77fmUeuO5Ofj/+c305cxMEjWsJEksocoIOZ5ZlZOpGEbWLJi8zsVKAJMLPEvU3MrLhJ7UJgcXD9bcAlwKDi+lMkVrVSU7j/sk48fE0X7n5pHn96bzlH1e2aMOKZzMUy7mMBcG1wPABoYGbNgs91gnEbn5jZ1VH35AM3BufeNrMO8QhekltBh+a8/ZO+bN93iCu1bpIkkaBF7S7gHWAJ8Iq7LzKzB80supdiEDDOowaJBt2t9wFTzGwhkS7bp4PTI4GWwMxg+MoDVfA6Us30O7UFb95dwMxV2/nBM7PY+rX2zk4EaXF8dizjPu4DHjezwcBHwEagePnpbHffZGbtgPfNbKG7rwRqAwfdvYeZXQOMJjJG5NtfbjYcGA6QnZ1dGe8jSaZxvXQev+ks3pi/kVtHz2ZoQR4jCvNJTSnt/5oiicPdJwGTSpQ9UOLzb8u4dzLQtZTyeNb3UoO0bFiHF247l8feX84Vj03nv68/k/M7anxlmOLZMlfuuA933+Tu17h7d+BXQdme4nPBP1cBHwDdo577anD8OqVUWsF9GsQrAPTvlsnEuwuYvnw7Nz41k3U7tISJiMjJSE0xfvpvHfnTwG78fPzn/Oc/vuTIUfXehyWeyVy54z7MrLmZFcfwSyKtbJhZEzOrXXwN0Idg3AeRAb0XBseFwLI4voNUE5mN6/LCbedw6RmtuPovM3hlrjaVFhE5Wb3zm/PmPQUs3vQ1A0d9wsbd34QdUo0Ut2QuxnEf/YClZraMyFiOh4Ly04C5ZraAyMSI37l7cTL3O+DaYDzIw2gVc4lRSopxW992vHj7OYyevpoRz3+qafYiIiepef3ajBnck4s6t6T/49OZvHhL2CHVOFYTWid69Ojhc+fODTsMSSCHio7yh3eXMWH+Rn53TVcu6NQi7JCqJTP71N17hB2HnBjVnVJRn67dyT0vzefi01ty/2WdqJ2WWv5N8h0VrTu1A4TUSLXTUvnl5afxp4Hd+fWEL/jV6ws5cLio/BtFRKRMZ+c05a17Cti46xuue3Imy7fsDTukGkHJnNRo57Zrxts/7cs3h49yxZ+nM3/97rBDEhFJao3rpfPULWdz3dlZDHr6E4aMmc1Hy7ZpnHIcaaq61HgN69TiDzd2463PN3Pbc3P4wbk53HVBe9JS9VtHROREmBm39s7lxp5tmTh/E/930hKKjjmDe+dyzVmZ1EtX+lGZ9F8rkcAVXVvz5t19+XTtLq4dOZPV2/eHHZKISFKrUyuVG3q25e2f9OXB/qfz4bJt9Pnd+zw8aYlmvlYiJXMiUVo1qsNzQ3oxoFsbrn3yY16YtVZdAyIiJ8nM6J3fnKd/2IM37iyg6JhzxZ+n8eMXPmXOmp2qZ0+SkjmRElJSjMF98njlR+fy0ux1DHtuLlv3assaEZHKkN2sHv/7ys5M/8WFnJPXjJ+P/5zvPz6dVz/dwKEi7aV9IpTMiZShfYsGvHZHH05r3YDv/f5D7nzxMyYv3sLhIq1yLiJysurXTuPW3rlM+VkhP7uoIxPmb6TgP6fy6ORl+gFdQVpnTiQGu/Yf5q2Fm5kwbyOrtu/nsjNaMaB7JmfnNMFMe72WRevMJTfVnVLVlm/Zy5iP1/Dmgk3822ktGdInjy5ZjcIOq8pVtO5UMidSQet3HmDigk28Pm8jB48cpX+3Ngzonkn7Fg3CDi3hKJlLbqo7JSy7Dxxm3Jz1jP14DZlN6jKkTx4Xd25ZY1YZUDJXClVIEg/uzqJNX/PG/I28MX8TGQ1qM6B7Jt8/sw0tG9YJO7yEoGQuuanulLAVHT3GO4u2MGbGajbvOcgt5+UwsGdbGtdLDzu0uFIyVwpVSBJvR485n6zawevzNvLuoq/oktWIq7tlcukZrWhQp1bY4YVGyVxyU90piWThhj2MmbGa95Zs4coz2zCkdy4dWlbPHhElc6VQhSRV6eCRo0xZspXX521k1qodnH9qBld3y6SwYwbpaTWji6CYkrnkprpTEtHWvQd54ZN1vDBrHae1bsDQPnkUdswgJaX6jF9WMlcKVUgSll37DzPpi8jEiRVb93F5l9Zc3T2Ts7ObVKuKpyxK5pKb6k5JZIeKjvI/CzYzZsZqvjl8lFt753Lt2VnUr538u0somSuFKiRJBMUTJybM28iBw0e5unsbru6WWW27CUDJXLJT3SnJwN2Zs2YXY2asZuaqHVx7VhaDe+fStmm9sEM7YUrmSqEKSRKJu7N489dMmLeRiQs20bx+ba7ulslV3arfxAklc8lNdackmw27DvC3mWt5Ze56euY2ZUifPM5t1zTplpBSMlcKVUiSqI4ec2YFEyfeCSZO9A8mTjSsBhMnlMydGDO7FPgTkAo84+6/K3H+UeCC4GM9oIW7Nw7OZQPPAG0BBy539zVmlgeMA5oCnwG3uPvh48WhulOS1YHDRbz62Ub+OmM1tVJTGNonj6u6taFOrdSwQ4uJkrlSqEKSZHDwyFHe/zIyceKTlTs4v2MG/bu1od+pLZJ24oSSuYozs1RgGXARsAGYAwxy98VlXH830N3dhwafPwAecvfJZlYfOObuB8zsFeA1dx9nZiOBBe7+5PFiUd0pye7YMWfaiu2MmbGaLzbuYWDPbG45Lyfhe0EqWncm/yhBkWqiTq1ULu/Smsu7tGb3gciOE89MW80vXv2cy7q05upumfTIqRkTJ2q4XsAKd18FYGbjgP5AqckcMAj4TXBtZyDN3ScDuPu+oNyAC4GbgnueA34LHDeZE0l2KSlGYccMCjtmsHLbPp77eA0XP/oRhR0zGFqQR7e2jcMOsVIomRNJQI3rpXPzOTncfE4OG3Yd4I35m/j1hIXsPxTZceLq7pl0rMYTJ2q4TGB91OcNwDmlXWhmOUAe8H5Q1BHYbWavBeXvAfcDTYDd7l4U9czMMp45HBgOkJ2dfVIvIpJI8jPq82D/M/hfF5/KK3PWc9eLn5HRoDZD+uRx2RmtqJXEu0somRNJcFlN6nHnBe35cb98lmzey4T5G/nhs7Npeko6V3dvw1VnZtKqUWJ3GUiFlNb0WtZ4mIHAeHc/GnxOA/oC3YF1wMvAYGBirM9091HAKIh0s8YctUiSaFS3Fref346hBXlMXhzZXeL/vrWEW87LYVCvbJqekny7SyiZE0kSZkbnNg3p3KYhv7i0E7NW72DCvI1c8sePOL1Nw8iOE12qx8SJGm4DkckLxbKATWVcOxC4s8S986K6aCcA5wKjgcZmlha0zh3vmSI1QmqKcekZrbj0jFYs2rSHv85YQ79HpnLZGa0ZUpBLp1YNww4xZsnbpihSg6WmGL3zm/Nf153JrP/4Hj84N4f3lmyhz8Pv8+MXPuXdRV9xuOhY2GHKiZkDdDCzPDNLJ5KwfadlzcxOJdJ9OrPEvU3MLCP4fCGw2CMz3aYC1wXltwJvxCl+kaRzeptGPHL9mbx/Xz+ymtTlh8/O5qanP2Hy4i0cPZb4DdRxnc0aw/T6HCK/GDOAncAP3H1DcO4osDC4dJ27X1Xi3seAIe5ev7w4NCNLaordBw4zaeFXTJi/keVb9nLpGa0Z0D28iROazXpizOxy4I9E6s7R7v6QmT0IzHX3icE1vwXquPv9Je69CPhvIt21nwLD3f2wmbXjX0uTzCNS3x46XhyqO6WmOlx0jEkLI7tL7DpwhFt753JDj6wq22s7YZYmiWV6vZn9HXjT3Z8zswuJJGe3BOf2lZWomVkP4CfAACVzIqXbsOtfO07sP3SUq7q1YUAVT5xQMpfcVHdKTefufLZuN2NmrGba8u0M6J7Jrb1zyWt+Sly/N5GWJollen1n4N7geCowobyHBkniI0Sm2A+ozIBFqpOsJvX4cb/23FEYmTjxRjBxoskp6QzQxAkRkXKZGWfnNOHsnCZs3vMNf5u5lmuf/JjubRszpE8efdo3S4jdJeKZzMUyvX4BcC2RrtgBQAMza+buO4A6ZjYXKAJ+5+7Fid5dwER335wI/wOKJLroiRM/DyZOvDFvE5f88SM6t27IgO6aOCEiUp7Wjery80s7cfeFHZgwfyMPvrkIgMG98xjQPZO66eHtLhHPZC6W6fX3AY+b2WDgI2AjkeQNINvdNwXjPN43s4XAN8D1QL9yv1xrJYl8R/HEid75zfn/+p/O1C+3MmH+Rv7Pm4vp27E5/btl0u/UDGqnJceWNyIiVa1ueiqDemUzsGdbPl65gzEzVvP7d5dyQ4+2/PC8HNo0rlvlMcVzzNx5wG/d/ZLg8y8B3P3hMq6vD3zp7lmlnPsr8CaRZO5Z4GBwKhtY5e7/r717j7GiPOM4/n0iywrFwlK0ZUFFxF4QdUW0WGxTLykIBDCxqe0/VJsmFZvUmqa10VD7B7FKmxLTpvamYG1RscSSWpISSkOjK0hxuRjLpSxNgRVKKbdSEdmnf7zvyuy657bsmTOz+/skk519z8w5v313zpP3zOXMuGJZdN6HSHFHTpziD1vbeOG1vWzbf4xbJ4xkTlMj144ZflYXTuicuXxT7RQpz+6D/2VJ826Wb9zLDZeN4K4pY5h4UUOPD8Fm6QKIAYQyay47AAAJLklEQVQLIG4m7HF7FfiCu7+eWGYEcMjd281sAXDa3eebWQNwwt1PxmWagdld701Y7CKJJBUkkfLtPfw/VrSECyeOn3yHWU2NzGkaxUc+VPmFExrM5Ztqp0hljr11imUb9rCkeTdDB9Vx55QxzLiiseL7a2dmMBfDFL283sxuBx4mHH5dC9wTB3CfAH4KtBO+C2+Ru/+ym+fXYE6kit5oO8oLLXtZ0bKPoYPquO3qUcxqamTk0PIOI2gwl2+qnSI9c7rdWfO3Azz5cis79h8Pt2ecfBEjhtSXtX6mBnNZoYIkcnba2511rYf4XcteVm59kwW3TWDmlY0l19NgLt9UO0XO3rY3j7H45VZe3NzG8nlTGHdByX1QGsx1RwVJpPe8deo07e4MHlj6+ikN5vJNtVOk9xw+8TZDB9WVdR5dlr5nTkT6oHPrdKWriEilhg0eWLXn1r1ZRURERHJMgzkRERGRHNNgTkRERCTHNJgTERERyTEN5kRERERyTIM5ERERkRzTYE5EREQkxzSYExEREckxDeZEREREcqxf3M7LzP4F/KOMRUcAB6scpxzKka0MoBxdlZvjYnc/v9phpDpUO3ObAZSjq7zlqKh29ovBXLnMbEMW7iOpHNnKoBzZzSHZkJXtIQs5spBBOfpfDh1mFREREckxDeZEREREckyDuc5+VusAkXKckYUMoBxdZSWHZENWtocs5MhCBlCOrvp0Dp0zJyIiIpJj2jMnIiIikmfu3qcm4AngALA10TYcWAXsiD8bYrsBjwE7gc3AxMQ6c+PyO4C5ifZrgC1xnceIezfLzPEQsBdoidP0xGPfjs+5DZiaaJ8W23YC9yfaLwHWxXzPAgML5LgQWAO8AbwOfC3tPimSIdX+AM4F1gObYo7vFlsXqI+/74yPj+lpvjJzLAZaE/3RVO3tNC57DvAa8Pta9IembEyodiYz1LxulsiRdn+odr43S6bqZs0LSG9PwKeAiXQuBI92dAhwP/BInJ8OrIz/8MnAusQbdlf82RDnO96064Hr4zorgVsryPEQ8I1ulh0fN876uEH8PW4o58T5scDAuMz4uM5zwB1x/nHg7gI5RnZswMB5wPb4eqn1SZEMqfZHzDckztcR3liTC60LzAMej/N3AM/2NF+ZORYDt3ezfNW207jsfcBvOFOUUu0PTdmYUO1MPm/N62aJHGn3h2rne587U3Wzzx1mdfe1wKEuzbOBJXF+CTAn0f6UB68Aw8xsJDAVWOXuh9z9P4RPYNPiY+9392YP/42nEs9VTo5CZgPPuPtJd28ljMavi9NOd9/l7m8DzwCzzcyAm4Dnu/mbuuZoc/eNcf4Y4RPeqDT7pEiGVPsj/k3H4691cfIi6yb76Hng5vhaFeWrIEex/qjKdmpmo4EZwC/i78X6sir9Idmg2tkpQ83rZokcafeHamdCFutmnxvMFfBBd2+D8OYALojto4B/JpbbE9uKte/ppr0SXzWzzWb2hJk19DDHB4DD7v5OJTnMbAxwNeHTTE36pEsGSLk/zOwcM2shHMZZRfgEVGjdd18vPn4kvlal+UrmcPeO/lgQ++OHZlbfw/6o5H+yCPgm0B5/L9aXVesPyax+XzuzUDe7yQGqnbWsnZmrm/1lMFeIddPmPWgv10+AS4EmoA34QVo5zGwI8FvgXnc/WmzRamXpJkPq/eHup929CRhN+AT0sSLrppbDzCYQzp/4KHAtYff/t6qZw8xmAgfc/a/J5iLrpv1+kezqF7UzC3WzQA7VzhrVzqzWzf4ymNsfd58Sfx6I7XsIJ5h2GA3sK9E+upv2srj7/rghtgM/J7whepLjIGF38YBycphZHaEQ/Nrdl8fmVPukuwy16o/42oeBPxPOoyi07ruvFx8fSjj8U2m+cnJMi4dU3N1PAk/S8/4odzudAswys92EXfk3ET5x1qw/JHP6be3MQt0slEO1s6a1M5t10zNw4m1vT8AYOp88u5DOJ60+Gudn0PnkyPV+5uTIVsKJkQ1xfnh87NW4bMfJkdMryDEyMf91wvFygMvpfCLkLsJJkAPi/CWcORHy8rjOMjqfbDmvQAYjHPdf1KU9tT4pkiHV/gDOB4bF+UHAX4CZhdYF7qHziavP9TRfmTlGJvprEfC9NLbTuPynOXMib6r9oSk7E6qdpWpWqv1RJIdqZwZqJxmqmzUvHr09AUsJu51PEUa4XyIcn15NuGR4deKfZsCPCcf+twCTEs9zF+GExJ3AnYn2ScDWuM6PKHzZcnc5fhVfZzOwgs5vyAfic24jcfUM4Wqc7fGxBxLtYwlX3eyMG1F9gRw3EHbRbiZxGXuafVIkQ6r9AVxJuJR8c8w7v9i6hMvgl8X29cDYnuYrM8efYn9sBZ7mzFVbVdtOCxSlVPtDUzYmVDuTGWpeN0vkUO3MQO0kQ3VTd4AQERERybH+cs6ciIiISJ+kwZyIiIhIjmkwJyIiIpJjGsyJiIiI5JgGcyIiIiI5psGc1JyZ3Wtmg2udQ0QkL1Q3JUlfTSI1F79Je5K7H6x1FhGRPFDdlCTtmZNUmdn7zOxFM9tkZlvN7DtAI7DGzNbEZT5jZs1mttHMlsX7EmJmu83sETNbH6dxsf2z8bk2mdna2v11IiK9T3VTStFgTtI2Ddjn7le5+wTCrVf2ATe6+41mNgJ4ELjF3ScCG4D7EusfdffrCN/MvSi2zQemuvtVwKy0/hARkZSobkpRGsxJ2rYAt8RPip909yNdHp8MjAdeMrMWYC5wceLxpYmf18f5l4DFZvZlwr3tRET6EtVNKWpArQNI/+Lu283sGsK95x42sz92WcSAVe7++UJP0XXe3b9iZh8n3FS5xcya3P3fvZ1dRKQWVDelFO2Zk1SZWSNwwt2fBr4PTASOAefFRV4BpiTO6xhsZh9OPMXnEj+b4zKXuvs6d58PHAQurP5fIiKSDtVNKUV75iRtVwALzawdOAXcTdjtv9LM2uL5H18ElppZfVznQWB7nK83s3WEDyIdn0IXmtllhE+nq4FN6fwpIiKpUN2UovTVJJIbuhRfRKQyqpv9gw6zioiIiOSY9syJiIiI5Jj2zImIiIjkmAZzIiIiIjmmwZyIiIhIjmkwJyIiIpJjGsyJiIiI5JgGcyIiIiI59n+1zi4zfSEjbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if EVALUATE_WHILE_TRAINING:\n",
    "    logs = evaluation_logger.get_log()\n",
    "    for i, (m, v) in enumerate(logs.items(), 1):\n",
    "        sb.glue(\"eval_{}\".format(m), v)\n",
    "        x = [save_checkpoints_steps*i for i in range(1, len(v)+1)]\n",
    "        plot.line_graph(\n",
    "            values=list(zip(v, x)),\n",
    "            labels=m,\n",
    "            x_name=\"steps\",\n",
    "            y_name=m,\n",
    "            subplot=(math.ceil(len(logs)/2), 2, i),\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.2 TensorBoard\n",
    "\n",
    "Once the train is done, you can browse the details of the training results as well as the metrics we logged from [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard).\n",
    "\n",
    "[]()|[]()|[]()\n",
    ":---:|:---:|:---:\n",
    "<img src=\"https://recodatasets.z20.web.core.windows.net/images/tensorboard_0.png?sanitize=true\"> |  <img src=\"https://recodatasets.z20.web.core.windows.net/images/tensorboard_1.png?sanitize=true\"> | <img src=\"https://recodatasets.z20.web.core.windows.net/images/tensorboard_2.png?sanitize=true\">\n",
    "\n",
    "To open the TensorBoard, open a terminal from the same directory of this notebook, run `tensorboard --logdir=model_checkpoints`, and open http://localhost:6006 from a browser.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Test and Export Model\n",
    "\n",
    "#### 4.1 Item rating prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmp_dvlimh6/model.ckpt-50000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/anaconda/envs/reco_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.0). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "application/papermill.record+json": {
       "rmse": 0.9525733983856536
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/papermill.record+json": {
       "mae": 0.7574306222200393
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rmse': 0.9525733983856536, 'mae': 0.7574306222200393}\n"
     ]
    }
   ],
   "source": [
    "if len(RATING_METRICS) > 0:\n",
    "    predictions = list(model.predict(input_fn=tf_utils.pandas_input_fn(df=test)))\n",
    "    prediction_df = test.drop(RATING_COL, axis=1)\n",
    "    prediction_df[PREDICT_COL] = [p['predictions'][0] for p in predictions]\n",
    "    \n",
    "    rating_results = {}\n",
    "    for m in RATING_METRICS:\n",
    "        result = evaluator.metrics[m](test, prediction_df, **cols)\n",
    "        sb.glue(m, result)\n",
    "        rating_results[m] = result\n",
    "    print(rating_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.2 Recommend k items\n",
    "For top-k recommendation evaluation, we use the ranking pool (all the user-item pairs) we prepared at the [training step](#ranking-pool). The difference is we remove users' seen items from the pool in this step which is more natural to the movie recommendation scenario."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmp_dvlimh6/model.ckpt-50000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/anaconda/envs/reco_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.0). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "application/papermill.record+json": {
       "ndcg_at_k": 0.0854092078835321
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/papermill.record+json": {
       "precision_at_k": 0.08504772004241784
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ndcg_at_k': 0.0854092078835321, 'precision_at_k': 0.08504772004241784}\n"
     ]
    }
   ],
   "source": [
    "if len(RANKING_METRICS) > 0:\n",
    "    predictions = list(model.predict(input_fn=tf_utils.pandas_input_fn(df=ranking_pool)))\n",
    "    prediction_df = ranking_pool.copy()\n",
    "    prediction_df[PREDICT_COL] = [p['predictions'][0] for p in predictions]\n",
    "\n",
    "    ranking_results = {}\n",
    "    for m in RANKING_METRICS:\n",
    "        result = evaluator.metrics[m](test, prediction_df, **{**cols, 'k': TOP_K})\n",
    "        sb.glue(m, result)\n",
    "        ranking_results[m] = result\n",
    "    print(ranking_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.3 Export Model\n",
    "Finally, we export the model so that we can load later for re-training, evaluation, and prediction.\n",
    "Examples of how to load, re-train, and evaluate the saved model can be found from [azureml_hyperdrive_wide_and_deep.ipynb](../04_model_select_and_optimize/azureml_hyperdrive_wide_and_deep.ipynb) notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(EXPORT_DIR_BASE, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/anaconda/envs/reco_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.0). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "application/papermill.record+json": {
       "saved_model_dir": "./outputs/model/1561848625"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model exported to ./outputs/model/1561848625\n"
     ]
    }
   ],
   "source": [
    "exported_path = tf_utils.export_model(\n",
    "    model=model,\n",
    "    train_input_fn=train_fn,\n",
    "    eval_input_fn=tf_utils.pandas_input_fn(\n",
    "        df=test, y_col=RATING_COL\n",
    "    ),\n",
    "    tf_feat_cols=wide_columns+deep_columns,\n",
    "    base_dir=EXPORT_DIR_BASE\n",
    ")\n",
    "sb.glue('saved_model_dir', str(exported_path))\n",
    "print(\"Model exported to\", str(exported_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Close the event file so that the model folder can be cleaned up.\n",
    "summary_writer = tf.summary.FileWriterCache.get(model.model_dir)\n",
    "summary_writer.close()\n",
    "\n",
    "# Cleanup temporary directory if used\n",
    "if TMP_DIR is not None:\n",
    "    TMP_DIR.cleanup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (reco_gpu)",
   "language": "python",
   "name": "reco_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}