3.5 - (VisRank) Evaluation procedure.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from models import VisRank\n",
    "from utils.data import extract_embedding\n",
    "from utils.metrics import (\n",
    "    auc_exact,\n",
    "    nDCG,\n",
    "    precision,\n",
    "    recall,\n",
    "    reciprocal_rank,\n",
    ")\n",
    "\n",
    "\n",
    "# Dataset\n",
    "DATASET = \"UGallery\"\n",
    "assert DATASET in [\"UGallery\", \"Wikimedia\"]\n",
    "\n",
    "# Parameters\n",
    "FEATURE_EXTRACTOR = \"resnet50\"\n",
    "assert FEATURE_EXTRACTOR in [\"resnet50\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mode\n",
    "MODE_PROFILE = \"user\"\n",
    "\n",
    "# Paths (general)\n",
    "EMBEDDING_PATH = os.path.join(\"data\", DATASET, f\"embedding-{FEATURE_EXTRACTOR}.npy\")\n",
    "EVALUATION_PATH = os.path.join(\"data\", DATASET, f\"naive-{MODE_PROFILE}-evaluation.csv\")\n",
    "\n",
    "# Paths (images)\n",
    "IMAGES_DIR = None\n",
    "if DATASET == \"Wikimedia\":\n",
    "    IMAGES_DIR = os.path.join(\"/\", \"mnt\", \"data2\", \"wikimedia\", \"imagenes_tarea\")\n",
    "elif DATASET == \"UGallery\":\n",
    "    IMAGES_DIR = os.path.join(\"/\", \"mnt\", \"workspace\", \"Ugallery\", \"mini-images-224-224-v2\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load embedding from file\n",
    "print(f\"\\nLoading embedding from file... ({EMBEDDING_PATH})\")\n",
    "embedding = np.load(EMBEDDING_PATH, allow_pickle=True)\n",
    "\n",
    "# Extract features and \"id2index\" mapping\n",
    "print(\"\\nExtracting data into variables...\")\n",
    "features, _, item_index2fn = extract_embedding(embedding, verbose=True)\n",
    "print(f\">> Features shape: {features.shape}\")\n",
    "del embedding  # Release some memory\n",
    "\n",
    "# Fallback for explicit_features\n",
    "explicit_features = np.copy(features)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load evaluation dataframe\n",
    "print(\"\\nLoad evaluation dataframe\")\n",
    "evaluation_df = pd.read_csv(EVALUATION_PATH)\n",
    "# Transform lists from str to int\n",
    "string_to_list = lambda s: list(map(int, s.split()))\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].apply(\n",
    "    lambda s: string_to_list(s) if isinstance(s, str) else s,\n",
    ")\n",
    "evaluation_df[\"predict\"] = evaluation_df[\"predict\"].apply(\n",
    "    lambda s: string_to_list(s) if isinstance(s, str) else s,\n",
    ")\n",
    "# Group evaluations by profile and user\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].map(tuple)\n",
    "evaluation_df = evaluation_df.groupby([\"profile\", \"user_id\"]).agg({\"predict\": sum}).reset_index()\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].map(list)\n",
    "print(f\">> Evaluation: {evaluation_df.shape}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model initialization\n",
    "print(\"\\nModel initialization\")\n",
    "model = VisRank(\n",
    "    features,  # Embedding\n",
    "    similarity_method=cosine_similarity,  # Similarity measure\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict all\n",
    "# If True, ranks every item including already consumed items\n",
    "# If False, ranks ALL - PROFILE (consumed) + PREDICT (ground truth)\n",
    "PREDICT_ALL = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# Metrics\n",
    "N_EVALS = len(evaluation_df.index)\n",
    "# Area Under the Curve (AUC)\n",
    "AUC = np.zeros(N_EVALS, dtype=float)\n",
    "# Reciprocal Rank (RR)\n",
    "RR = np.zeros(N_EVALS, dtype=float)\n",
    "# Recall\n",
    "R20 = np.zeros(N_EVALS, dtype=float)\n",
    "R100 = np.zeros(N_EVALS, dtype=float)\n",
    "R200 = np.zeros(N_EVALS, dtype=float)\n",
    "# Precision\n",
    "P20 = np.zeros(N_EVALS, dtype=float)\n",
    "P100 = np.zeros(N_EVALS, dtype=float)\n",
    "P200 = np.zeros(N_EVALS, dtype=float)\n",
    "# Normalized discounted cumulative gain (nDCG)\n",
    "N20 = np.zeros(N_EVALS, dtype=float)\n",
    "N100 = np.zeros(N_EVALS, dtype=float)\n",
    "N200 = np.zeros(N_EVALS, dtype=float)\n",
    "PROFILE_SIZES = np.zeros(N_EVALS, dtype=int)\n",
    "N_ITEMS = len(features)\n",
    "\n",
    "\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].map(tuple)\n",
    "grouped_evals = evaluation_df.groupby([\"profile\", \"user_id\"]).agg({\"predict\": sum}).reset_index()\n",
    "for i, row in tqdm(enumerate(evaluation_df.itertuples()), total=len(evaluation_df.index)):\n",
    "    # Load data into tensors\n",
    "    profile = np.array(row.profile)\n",
    "    user_id = int(row.user_id)\n",
    "    predict = row.predict\n",
    "    # Prediction\n",
    "    indexes, _ = model.most_similar_to_profile(profile, k=None, method=\"maximum\", include_consumed=True)\n",
    "    if not PREDICT_ALL:\n",
    "        indexes = np.delete(\n",
    "            indexes,\n",
    "            np.where(np.isin(indexes, profile) & ~np.isin(indexes, predict)),\n",
    "        )\n",
    "    # Ranking\n",
    "    pos_of_evals = torch.Tensor(np.where(np.isin(indexes, predict))).flatten()\n",
    "    # Store metrics\n",
    "    AUC[i] = auc_exact(pos_of_evals, N_ITEMS)\n",
    "    RR[i] = reciprocal_rank(pos_of_evals)\n",
    "    R20[i] = recall(pos_of_evals, 20)\n",
    "    P20[i] = precision(pos_of_evals, 20)\n",
    "    N20[i] = nDCG(pos_of_evals, 20)\n",
    "    R100[i] = recall(pos_of_evals, 100)\n",
    "    P100[i] = precision(pos_of_evals, 100)\n",
    "    N100[i] = nDCG(pos_of_evals, 100)\n",
    "    R200[i] = recall(pos_of_evals, 200)\n",
    "    P200[i] = precision(pos_of_evals, 200)\n",
    "    N200[i] = nDCG(pos_of_evals, 200)\n",
    "    PROFILE_SIZES[i] = len(row.profile)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display stats\n",
    "print(f\"AVG AUC = {AUC.mean()}\")\n",
    "print(f\"AVG RR = {RR.mean()}\")\n",
    "print(f\"AVG R20 = {R20.mean()}\")\n",
    "print(f\"AVG P20 = {P20.mean()}\")\n",
    "print(f\"AVG NDCG20 = {N20.mean()}\")\n",
    "print(f\"AVG R100 = {R100.mean()}\")\n",
    "print(f\"AVG P100 = {P100.mean()}\")\n",
    "print(f\"AVG NDCG100 = {N100.mean()}\")\n",
    "print(f\"AVG R200 = {R200.mean()}\")\n",
    "print(f\"AVG P200 = {P200.mean()}\")\n",
    "print(f\"AVG NDCG200 = {N200.mean()}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results inspection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_ROW = 1\n",
    "\n",
    "assert 0 <= USER_ROW < len(evaluation_df)\n",
    "\n",
    "\n",
    "# Row in evaluation dataframe\n",
    "row = evaluation_df.iloc[USER_ROW]\n",
    "\n",
    "# Load data into tensors\n",
    "profile = np.array(row.profile, ndmin=1)\n",
    "user_id = int(row.user_id)\n",
    "predict = np.array(row.predict, ndmin=1)\n",
    "# Prediction\n",
    "indexes, _ = model.most_similar_to_profile(profile, k=None, method=\"maximum\", include_consumed=True)\n",
    "if not PREDICT_ALL:\n",
    "    indexes = np.delete(\n",
    "        indexes,\n",
    "        np.where(np.isin(indexes, profile) & ~np.isin(indexes, predict)),\n",
    "    )\n",
    "# Ranking\n",
    "pos_of_evals = torch.Tensor(np.where(np.isin(indexes, predict))).flatten()\n",
    "\n",
    "# Display metrics\n",
    "print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "print(f\"| {'Metric':^15} | {'Score':^7} |\")\n",
    "print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "print(f\"| {'AUC':^15} | {auc_exact(pos_of_evals, N_ITEMS):.5f} |\")\n",
    "print(f\"| {'RR':^15} | {reciprocal_rank(pos_of_evals):.5f} |\")\n",
    "for k in [20, 100, 500]:\n",
    "    print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "    print(f\"| {f'Recall@{k}':^15} | {recall(pos_of_evals, k):.5f} |\")\n",
    "    print(f\"| {f'Precision@{k}':^15} | {precision(pos_of_evals, k):.5f} |\")\n",
    "    print(f\"| {f'nDCG@{k}':^15} | {nDCG(pos_of_evals, k):.5f} |\")\n",
    "print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "\n",
    "# Ranking\n",
    "K = 20\n",
    "ranking = indexes\n",
    "if not PREDICT_ALL:\n",
    "    ranking = ranking[(~np.isin(ranking, profile)) | (np.isin(ranking, predict))]\n",
    "ranking = ranking[:K]\n",
    "print()\n",
    "print(f\"Size of profile: {profile.size}\")\n",
    "print(f\"Position of actual items: {pos_of_evals.cpu().numpy()}\")\n",
    "\n",
    "\n",
    "\n",
    "COLUMNS = 10\n",
    "ELEMENTS = {\n",
    "    \"Consumed\": profile,\n",
    "    \"Recommendation\": ranking,\n",
    "    \"Ground truth\": predict,\n",
    "}\n",
    "SHOW_FILENAME = False\n",
    "\n",
    "for label, items in ELEMENTS.items():\n",
    "    n_rows = ((len(items) - 1) // COLUMNS + 1)\n",
    "    fig = plt.figure(figsize=(COLUMNS * 2, 4 * n_rows))\n",
    "    plt.title(f\"{label.title()} (n={len(items)})\")\n",
    "    plt.axis(\"off\")\n",
    "    for i, img_id in enumerate(items, start=1):\n",
    "        img_fn = item_index2fn[img_id]\n",
    "        image = mpimg.imread(os.path.join(IMAGES_DIR, img_fn))\n",
    "        ax = fig.add_subplot(n_rows, COLUMNS, i)\n",
    "        if SHOW_FILENAME:\n",
    "            ax.set_title(img_fn)\n",
    "        if label == \"Recommendation\":\n",
    "            if img_id in predict:\n",
    "                ax.patch.set_edgecolor(\"green\")\n",
    "                ax.patch.set_linewidth(\"5\")\n",
    "                if SHOW_FILENAME:\n",
    "                    ax.set_title(img_fn, color=\"green\")\n",
    "                else:\n",
    "                    ax.set_title(\"Ground truth\", color=\"green\")\n",
    "            elif img_id in profile:\n",
    "                ax.patch.set_edgecolor(\"red\")\n",
    "                ax.patch.set_linewidth(\"5\")\n",
    "                if SHOW_FILENAME:\n",
    "                    ax.set_title(img_fn, color=\"red\")\n",
    "                else:\n",
    "                    ax.set_title(\"Consumed\", color=\"red\")\n",
    "        plt.xticks([])\n",
    "        plt.yticks([])\n",
    "        plt.imshow(image)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "3.8.5",
   "language": "python",
   "name": "3.8.5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}