analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "8b8228e4-ee26-4998-b94b-a9bcb5ef0ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from make_df import make_df_from_spreadsheet\n",
    "from rectangle_model import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fb52ecd3-0413-41aa-8864-dda687fb9190",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"exp1_data_clean.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "df4174f2-4cea-46b3-ba15-c29a52fc9771",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker</th>\n",
       "      <th>problem</th>\n",
       "      <th>example</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>37</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>37</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            worker  problem  example  row  col\n",
       "0   A28AXX4NCWPH1F       37        1    1    6\n",
       "1   A28AXX4NCWPH1F       37        2    1    1\n",
       "2   A28AXX4NCWPH1F       37        3    6    1\n",
       "3   A28AXX4NCWPH1F       37        4    6    6\n",
       "4   A28AXX4NCWPH1F       24        1    1    3\n",
       "5   A28AXX4NCWPH1F       24        2    6    3\n",
       "6   A28AXX4NCWPH1F       24        3    3    2\n",
       "7   A28AXX4NCWPH1F       24        4    3    5\n",
       "8   A28AXX4NCWPH1F       11        1    2    3\n",
       "9   A28AXX4NCWPH1F       11        2    6    1\n",
       "10  A28AXX4NCWPH1F       11        3    6    6\n",
       "11  A28AXX4NCWPH1F       11        4    2    4\n",
       "12  A28AXX4NCWPH1F        5        1    5    1\n",
       "13  A28AXX4NCWPH1F        5        2    5    6\n",
       "14  A28AXX4NCWPH1F        2        1    1    2"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8333f60-432e-4457-9c5b-e5c995146781",
   "metadata": {},
   "source": [
    "Change to zero indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "92aa9642-b6f1-44c7-9efe-7d3f6effd624",
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_change = df.columns[1:]\n",
    "for c in cols_to_change: \n",
    "    df[c] = df[c] - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bc3b12ca-2c1c-45b6-b12f-31d2c82f0262",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['coords'] = list(zip(df['row'], df['col']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b7fc7c20-f8d6-4c76-b749-4330140cea67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker</th>\n",
       "      <th>problem</th>\n",
       "      <th>example</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "      <th>coords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>(0, 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>(5, 0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>(5, 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>(0, 2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2582</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>(4, 3)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2583</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>(2, 0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2584</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>(3, 0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2585</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>(2, 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2586</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>(3, 5)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2587 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              worker  problem  example  row  col  coords\n",
       "0     A28AXX4NCWPH1F       36        0    0    5  (0, 5)\n",
       "1     A28AXX4NCWPH1F       36        1    0    0  (0, 0)\n",
       "2     A28AXX4NCWPH1F       36        2    5    0  (5, 0)\n",
       "3     A28AXX4NCWPH1F       36        3    5    5  (5, 5)\n",
       "4     A28AXX4NCWPH1F       23        0    0    2  (0, 2)\n",
       "...              ...      ...      ...  ...  ...     ...\n",
       "2582  A3HNEYFOIJWPH1       21        1    4    3  (4, 3)\n",
       "2583  A3HNEYFOIJWPH1        0        0    2    0  (2, 0)\n",
       "2584  A3HNEYFOIJWPH1        0        1    3    0  (3, 0)\n",
       "2585  A3HNEYFOIJWPH1        0        2    2    5  (2, 5)\n",
       "2586  A3HNEYFOIJWPH1        0        3    3    5  (3, 5)\n",
       "\n",
       "[2587 rows x 6 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "56ba733f-13b3-4083-abd3-0ca7de52210a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to new coords system\n",
    "\n",
    "def flatten_coords(coords): \n",
    "    \"\"\"input is coords as a tuple, output is 0 to 36 index for coords\"\"\"\n",
    "    return 6*coords[0] + coords[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "20105729-c8f7-49d1-a243-60f0135b3438",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['coords_idx'] = df['coords'].apply(flatten_coords)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "2a02e71e-043a-4a1f-b0b7-e5e78bb254aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker</th>\n",
       "      <th>problem</th>\n",
       "      <th>example</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "      <th>coords</th>\n",
       "      <th>coords_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>(0, 5)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>(5, 0)</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>(5, 5)</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>(0, 2)</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2582</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>(4, 3)</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2583</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>(2, 0)</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2584</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>(3, 0)</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2585</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>(2, 5)</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2586</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>(3, 5)</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2587 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              worker  problem  example  row  col  coords  coords_idx\n",
       "0     A28AXX4NCWPH1F       36        0    0    5  (0, 5)           5\n",
       "1     A28AXX4NCWPH1F       36        1    0    0  (0, 0)           0\n",
       "2     A28AXX4NCWPH1F       36        2    5    0  (5, 0)          30\n",
       "3     A28AXX4NCWPH1F       36        3    5    5  (5, 5)          35\n",
       "4     A28AXX4NCWPH1F       23        0    0    2  (0, 2)           2\n",
       "...              ...      ...      ...  ...  ...     ...         ...\n",
       "2582  A3HNEYFOIJWPH1       21        1    4    3  (4, 3)          27\n",
       "2583  A3HNEYFOIJWPH1        0        0    2    0  (2, 0)          12\n",
       "2584  A3HNEYFOIJWPH1        0        1    3    0  (3, 0)          18\n",
       "2585  A3HNEYFOIJWPH1        0        2    2    5  (2, 5)          17\n",
       "2586  A3HNEYFOIJWPH1        0        3    3    5  (3, 5)          23\n",
       "\n",
       "[2587 rows x 7 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "012df284-8583-42cd-b7fd-7a68b9d86f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = df.groupby(['worker', 'problem'])['coords_idx'].apply(tuple).reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cafa885-86e9-4793-8ec1-1e1987c77e8e",
   "metadata": {},
   "source": [
    "new_df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2053b74c-d99d-4a5e-8c19-69648f7434a0",
   "metadata": {},
   "source": [
    "Add model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "96055c20-a193-475f-8875-27fa7a674ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'teaching_stimuli - all_examples (9).csv'\n",
    "all_problems = make_df_from_spreadsheet(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "efd72ae2-0949-4ba3-a61f-2e3640cf166e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate model predictions for our problems\n",
    "\n",
    "# Add a column for old problem indices ...\n",
    "old_prob_indices = [43, 47, 55, 57, 58, 59, 60, 61, 62, 63, 64, 65] + [i for i in range(66, 94)] \n",
    "new_df['old_prob_idx'] = new_df['problem'].apply(lambda x : old_prob_indices[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "6504b508-3b06-4fb8-b36c-3721bb459457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker</th>\n",
       "      <th>problem</th>\n",
       "      <th>example</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "      <th>coords</th>\n",
       "      <th>coords_idx</th>\n",
       "      <th>old_prob_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>(0, 5)</td>\n",
       "      <td>5</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>(5, 0)</td>\n",
       "      <td>30</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>(5, 5)</td>\n",
       "      <td>35</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>(0, 2)</td>\n",
       "      <td>2</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2582</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>(4, 3)</td>\n",
       "      <td>27</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2583</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>(2, 0)</td>\n",
       "      <td>12</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2584</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>(3, 0)</td>\n",
       "      <td>18</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2585</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>(2, 5)</td>\n",
       "      <td>17</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2586</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>(3, 5)</td>\n",
       "      <td>23</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2587 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              worker  problem  example  row  col  coords  coords_idx  \\\n",
       "0     A28AXX4NCWPH1F       36        0    0    5  (0, 5)           5   \n",
       "1     A28AXX4NCWPH1F       36        1    0    0  (0, 0)           0   \n",
       "2     A28AXX4NCWPH1F       36        2    5    0  (5, 0)          30   \n",
       "3     A28AXX4NCWPH1F       36        3    5    5  (5, 5)          35   \n",
       "4     A28AXX4NCWPH1F       23        0    0    2  (0, 2)           2   \n",
       "...              ...      ...      ...  ...  ...     ...         ...   \n",
       "2582  A3HNEYFOIJWPH1       21        1    4    3  (4, 3)          27   \n",
       "2583  A3HNEYFOIJWPH1        0        0    2    0  (2, 0)          12   \n",
       "2584  A3HNEYFOIJWPH1        0        1    3    0  (3, 0)          18   \n",
       "2585  A3HNEYFOIJWPH1        0        2    2    5  (2, 5)          17   \n",
       "2586  A3HNEYFOIJWPH1        0        3    3    5  (3, 5)          23   \n",
       "\n",
       "      old_prob_idx  \n",
       "0               90  \n",
       "1               90  \n",
       "2               90  \n",
       "3               90  \n",
       "4               77  \n",
       "...            ...  \n",
       "2582            75  \n",
       "2583            43  \n",
       "2584            43  \n",
       "2585            43  \n",
       "2586            43  \n",
       "\n",
       "[2587 rows x 8 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "66550d9a-912d-4af8-a664-ad22acd4cdba",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-77-f1bc016c00aa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mmodel_preds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mmodel_preds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lit'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_teacher_probs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_problems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mmodel_preds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'prag'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_teacher_probs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m250\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_problems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Dropbox/teaching_models/rectangle_model.py\u001b[0m in \u001b[0;36mfind_teacher_probs\u001b[0;34m(n_iter, prob_idx, all_problems)\u001b[0m\n\u001b[1;32m    564\u001b[0m     \u001b[0mmy_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'd'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmy_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'h'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_teacher_probs_k4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprob_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_problems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    565\u001b[0m     \u001b[0mmy_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'd'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmy_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'h'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_teacher_probs_k5\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprob_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_problems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 566\u001b[0;31m     \u001b[0mmy_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'd'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmy_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'h'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_teacher_probs_k6\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprob_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_problems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    568\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmy_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Dropbox/teaching_models/rectangle_model.py\u001b[0m in \u001b[0;36mfind_teacher_probs_k6\u001b[0;34m(n_iter, prob_idx, all_problems)\u001b[0m\n\u001b[1;32m    473\u001b[0m     \"\"\"\n\u001b[1;32m    474\u001b[0m     \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh_flat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_problem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprob_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_problems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m     \u001b[0mdf_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfill_df_6ex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh_flat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m     \u001b[0mdf_d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_h\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_teacher_probabilities_given_iter_0\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Dropbox/teaching_models/rectangle_model.py\u001b[0m in \u001b[0;36mfill_df_6ex\u001b[0;34m(h_flat)\u001b[0m\n\u001b[1;32m    380\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcolumn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf_4ex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf_4ex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_d_possible\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_d_possible\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_d_possible\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_d_possible\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mm\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_d_possible\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_d_possible\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m                 \u001b[0mdf_4ex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# loop over old prob index and create dataframes for model predictions for each \n",
    "model_preds = {}\n",
    "\n",
    "for idx in old_prob_indices: \n",
    "    model_preds[idx] = {}\n",
    "    model_preds[idx]['lit'] = find_teacher_probs(0, idx, all_problems)\n",
    "    model_preds[idx]['prag'] = find_teacher_probs(250, idx, all_problems)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "f9c86201-d50b-415b-b5b0-22c7a714c03c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{43: {'lit': {'n_iter': 0,\n",
       "   'problem_index': 43,\n",
       "   1: {'d':          h_1       h_2       h_3       h_4\n",
       "    0   0.000000  0.153846  0.000000  0.000000\n",
       "    1   0.000000  0.153846  0.000000  0.000000\n",
       "    2   0.045455  0.076923  0.000000  0.000000\n",
       "    3   0.045455  0.000000  0.076923  0.000000\n",
       "    4   0.000000  0.000000  0.153846  0.000000\n",
       "    5   0.000000  0.000000  0.153846  0.000000\n",
       "    6   0.000000  0.153846  0.000000  0.000000\n",
       "    7   0.045455  0.076923  0.000000  0.000000\n",
       "    8   0.045455  0.076923  0.000000  0.000000\n",
       "    9   0.045455  0.000000  0.076923  0.000000\n",
       "    10  0.045455  0.000000  0.076923  0.000000\n",
       "    11  0.000000  0.000000  0.153846  0.000000\n",
       "    12  0.045455  0.076923  0.000000  0.000000\n",
       "    13  0.045455  0.076923  0.000000  0.000000\n",
       "    14  0.022727  0.038462  0.038462  0.020833\n",
       "    15  0.022727  0.038462  0.038462  0.020833\n",
       "    16  0.045455  0.000000  0.076923  0.000000\n",
       "    17  0.045455  0.000000  0.076923  0.000000\n",
       "    18  0.045455  0.000000  0.000000  0.041667\n",
       "    19  0.045455  0.000000  0.000000  0.041667\n",
       "    20  0.022727  0.038462  0.038462  0.020833\n",
       "    21  0.022727  0.038462  0.038462  0.020833\n",
       "    22  0.045455  0.000000  0.000000  0.041667\n",
       "    23  0.045455  0.000000  0.000000  0.041667\n",
       "    24  0.000000  0.000000  0.000000  0.083333\n",
       "    25  0.045455  0.000000  0.000000  0.041667\n",
       "    26  0.045455  0.000000  0.000000  0.041667\n",
       "    27  0.045455  0.000000  0.000000  0.041667\n",
       "    28  0.045455  0.000000  0.000000  0.041667\n",
       "    29  0.000000  0.000000  0.000000  0.083333\n",
       "    30  0.000000  0.000000  0.000000  0.083333\n",
       "    31  0.000000  0.000000  0.000000  0.083333\n",
       "    32  0.045455  0.000000  0.000000  0.041667\n",
       "    33  0.045455  0.000000  0.000000  0.041667\n",
       "    34  0.000000  0.000000  0.000000  0.083333\n",
       "    35  0.000000  0.000000  0.000000  0.083333,\n",
       "    'h':      h_1   h_2   h_3   h_4\n",
       "    0   0.00  1.00  0.00  0.00\n",
       "    1   0.00  1.00  0.00  0.00\n",
       "    2   0.50  0.50  0.00  0.00\n",
       "    3   0.50  0.00  0.50  0.00\n",
       "    4   0.00  0.00  1.00  0.00\n",
       "    5   0.00  0.00  1.00  0.00\n",
       "    6   0.00  1.00  0.00  0.00\n",
       "    7   0.50  0.50  0.00  0.00\n",
       "    8   0.50  0.50  0.00  0.00\n",
       "    9   0.50  0.00  0.50  0.00\n",
       "    10  0.50  0.00  0.50  0.00\n",
       "    11  0.00  0.00  1.00  0.00\n",
       "    12  0.50  0.50  0.00  0.00\n",
       "    13  0.50  0.50  0.00  0.00\n",
       "    14  0.25  0.25  0.25  0.25\n",
       "    15  0.25  0.25  0.25  0.25\n",
       "    16  0.50  0.00  0.50  0.00\n",
       "    17  0.50  0.00  0.50  0.00\n",
       "    18  0.50  0.00  0.00  0.50\n",
       "    19  0.50  0.00  0.00  0.50\n",
       "    20  0.25  0.25  0.25  0.25\n",
       "    21  0.25  0.25  0.25  0.25\n",
       "    22  0.50  0.00  0.00  0.50\n",
       "    23  0.50  0.00  0.00  0.50\n",
       "    24  0.00  0.00  0.00  1.00\n",
       "    25  0.50  0.00  0.00  0.50\n",
       "    26  0.50  0.00  0.00  0.50\n",
       "    27  0.50  0.00  0.00  0.50\n",
       "    28  0.50  0.00  0.00  0.50\n",
       "    29  0.00  0.00  0.00  1.00\n",
       "    30  0.00  0.00  0.00  1.00\n",
       "    31  0.00  0.00  0.00  1.00\n",
       "    32  0.50  0.00  0.00  0.50\n",
       "    33  0.50  0.00  0.00  0.50\n",
       "    34  0.00  0.00  0.00  1.00\n",
       "    35  0.00  0.00  0.00  1.00},\n",
       "   2: {'d':          h_1       h_2  h_3       h_4\n",
       "    i_1 i_2                              \n",
       "    0   1    0.0  0.021505  0.0  0.000000\n",
       "        2    0.0  0.021505  0.0  0.000000\n",
       "        3    0.0  0.000000  0.0  0.000000\n",
       "        4    0.0  0.000000  0.0  0.000000\n",
       "        5    0.0  0.000000  0.0  0.000000\n",
       "    ...      ...       ...  ...       ...\n",
       "    32  34   0.0  0.000000  0.0  0.006993\n",
       "        35   0.0  0.000000  0.0  0.006993\n",
       "    33  34   0.0  0.000000  0.0  0.006993\n",
       "        35   0.0  0.000000  0.0  0.006993\n",
       "    34  35   0.0  0.000000  0.0  0.006993\n",
       "    \n",
       "    [630 rows x 4 columns],\n",
       "    'h':          h_1  h_2  h_3  h_4\n",
       "    i_1 i_2                    \n",
       "    0   1    0.0  1.0  0.0  0.0\n",
       "        2    0.0  1.0  0.0  0.0\n",
       "        3    0.0  0.0  0.0  0.0\n",
       "        4    0.0  0.0  0.0  0.0\n",
       "        5    0.0  0.0  0.0  0.0\n",
       "    ...      ...  ...  ...  ...\n",
       "    32  34   0.0  0.0  0.0  1.0\n",
       "        35   0.0  0.0  0.0  1.0\n",
       "    33  34   0.0  0.0  0.0  1.0\n",
       "        35   0.0  0.0  0.0  1.0\n",
       "    34  35   0.0  0.0  0.0  1.0\n",
       "    \n",
       "    [630 rows x 4 columns]},\n",
       "   3: {'d':              h_1      h_2  h_3       h_4\n",
       "    i_1 i_2 i_3                             \n",
       "    0   1   2    0.0  0.00565  0.0  0.000000\n",
       "            3    0.0  0.00000  0.0  0.000000\n",
       "            4    0.0  0.00000  0.0  0.000000\n",
       "            5    0.0  0.00000  0.0  0.000000\n",
       "            6    0.0  0.00565  0.0  0.000000\n",
       "    ...          ...      ...  ...       ...\n",
       "    31  34  35   0.0  0.00000  0.0  0.001045\n",
       "    32  33  34   0.0  0.00000  0.0  0.001045\n",
       "            35   0.0  0.00000  0.0  0.001045\n",
       "        34  35   0.0  0.00000  0.0  0.001045\n",
       "    33  34  35   0.0  0.00000  0.0  0.001045\n",
       "    \n",
       "    [7140 rows x 4 columns],\n",
       "    'h':              h_1  h_2  h_3  h_4\n",
       "    i_1 i_2 i_3                    \n",
       "    0   1   2    0.0  1.0  0.0  0.0\n",
       "            3    0.0  0.0  0.0  0.0\n",
       "            4    0.0  0.0  0.0  0.0\n",
       "            5    0.0  0.0  0.0  0.0\n",
       "            6    0.0  1.0  0.0  0.0\n",
       "    ...          ...  ...  ...  ...\n",
       "    31  34  35   0.0  0.0  0.0  1.0\n",
       "    32  33  34   0.0  0.0  0.0  1.0\n",
       "            35   0.0  0.0  0.0  1.0\n",
       "        34  35   0.0  0.0  0.0  1.0\n",
       "    33  34  35   0.0  0.0  0.0  1.0\n",
       "    \n",
       "    [7140 rows x 4 columns]},\n",
       "   4: {'d':                  h_1       h_2  h_3      h_4\n",
       "    i_1 i_2 i_3 i_4                             \n",
       "    0   1   2   3    0.0  0.000000  0.0  0.00000\n",
       "                4    0.0  0.000000  0.0  0.00000\n",
       "                5    0.0  0.000000  0.0  0.00000\n",
       "                6    0.0  0.002316  0.0  0.00000\n",
       "                7    0.0  0.002316  0.0  0.00000\n",
       "    ...              ...       ...  ...      ...\n",
       "    31  32  33  34   0.0  0.000000  0.0  0.00023\n",
       "                35   0.0  0.000000  0.0  0.00023\n",
       "            34  35   0.0  0.000000  0.0  0.00023\n",
       "        33  34  35   0.0  0.000000  0.0  0.00023\n",
       "    32  33  34  35   0.0  0.000000  0.0  0.00023\n",
       "    \n",
       "    [58905 rows x 4 columns],\n",
       "    'h':                  h_1  h_2  h_3  h_4\n",
       "    i_1 i_2 i_3 i_4                    \n",
       "    0   1   2   3    0.0  0.0  0.0  0.0\n",
       "                4    0.0  0.0  0.0  0.0\n",
       "                5    0.0  0.0  0.0  0.0\n",
       "                6    0.0  1.0  0.0  0.0\n",
       "                7    0.0  1.0  0.0  0.0\n",
       "    ...              ...  ...  ...  ...\n",
       "    31  32  33  34   0.0  0.0  0.0  1.0\n",
       "                35   0.0  0.0  0.0  1.0\n",
       "            34  35   0.0  0.0  0.0  1.0\n",
       "        33  34  35   0.0  0.0  0.0  1.0\n",
       "    32  33  34  35   0.0  0.0  0.0  1.0\n",
       "    \n",
       "    [58905 rows x 4 columns]},\n",
       "   5: {'d':                      h_1  h_2  h_3       h_4\n",
       "    i_1 i_2 i_3 i_4 i_5                         \n",
       "    0   1   2   3   4    0.0  0.0  0.0  0.000000\n",
       "                    5    0.0  0.0  0.0  0.000000\n",
       "                    6    0.0  0.0  0.0  0.000000\n",
       "                    7    0.0  0.0  0.0  0.000000\n",
       "                    8    0.0  0.0  0.0  0.000000\n",
       "    ...                  ...  ...  ...       ...\n",
       "    30  31  32  33  35   0.0  0.0  0.0  0.000069\n",
       "                34  35   0.0  0.0  0.0  0.000069\n",
       "            33  34  35   0.0  0.0  0.0  0.000069\n",
       "        32  33  34  35   0.0  0.0  0.0  0.000069\n",
       "    31  32  33  34  35   0.0  0.0  0.0  0.000069\n",
       "    \n",
       "    [376992 rows x 4 columns],\n",
       "    'h':                      h_1  h_2  h_3  h_4\n",
       "    i_1 i_2 i_3 i_4 i_5                    \n",
       "    0   1   2   3   4    0.0  0.0  0.0  0.0\n",
       "                    5    0.0  0.0  0.0  0.0\n",
       "                    6    0.0  0.0  0.0  0.0\n",
       "                    7    0.0  0.0  0.0  0.0\n",
       "                    8    0.0  0.0  0.0  0.0\n",
       "    ...                  ...  ...  ...  ...\n",
       "    30  31  32  33  35   0.0  0.0  0.0  1.0\n",
       "                34  35   0.0  0.0  0.0  1.0\n",
       "            33  34  35   0.0  0.0  0.0  1.0\n",
       "        32  33  34  35   0.0  0.0  0.0  1.0\n",
       "    31  32  33  34  35   0.0  0.0  0.0  1.0\n",
       "    \n",
       "    [376992 rows x 4 columns]},\n",
       "   6: {'d':                          h_1  h_2  h_3       h_4\n",
       "    i_1 i_2 i_3 i_4 i_5 i_6                         \n",
       "    0   1   2   3   4   5    0.0  0.0  0.0  0.000000\n",
       "                        6    0.0  0.0  0.0  0.000000\n",
       "                        7    0.0  0.0  0.0  0.000000\n",
       "                        8    0.0  0.0  0.0  0.000000\n",
       "                        9    0.0  0.0  0.0  0.000000\n",
       "    ...                      ...  ...  ...       ...\n",
       "    29  30  31  32  34  35   0.0  0.0  0.0  0.000027\n",
       "                33  34  35   0.0  0.0  0.0  0.000027\n",
       "            32  33  34  35   0.0  0.0  0.0  0.000027\n",
       "        31  32  33  34  35   0.0  0.0  0.0  0.000027\n",
       "    30  31  32  33  34  35   0.0  0.0  0.0  0.000027\n",
       "    \n",
       "    [1947792 rows x 4 columns],\n",
       "    'h':                          h_1  h_2  h_3  h_4\n",
       "    i_1 i_2 i_3 i_4 i_5 i_6                    \n",
       "    0   1   2   3   4   5    0.0  0.0  0.0  0.0\n",
       "                        6    0.0  0.0  0.0  0.0\n",
       "                        7    0.0  0.0  0.0  0.0\n",
       "                        8    0.0  0.0  0.0  0.0\n",
       "                        9    0.0  0.0  0.0  0.0\n",
       "    ...                      ...  ...  ...  ...\n",
       "    29  30  31  32  34  35   0.0  0.0  0.0  1.0\n",
       "                33  34  35   0.0  0.0  0.0  1.0\n",
       "            32  33  34  35   0.0  0.0  0.0  1.0\n",
       "        31  32  33  34  35   0.0  0.0  0.0  1.0\n",
       "    30  31  32  33  34  35   0.0  0.0  0.0  1.0\n",
       "    \n",
       "    [1947792 rows x 4 columns]}}}}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_preds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "537ef245-67d2-49cd-a8ae-8c192b95b7b9",
   "metadata": {},
   "source": [
    "### adlsfkjaldfk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0f8bbe14-f259-4e88-afe8-81a76018161a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-60-79700faa422c>:2: PerformanceWarning: indexing past lexsort depth may impact performance.\n",
      "  temp_df.loc[('A28AXX4NCWPH1F', 36)][:]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>example</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "      <th>coords</th>\n",
       "      <th>coords_idx</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worker</th>\n",
       "      <th>problem</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">A28AXX4NCWPH1F</th>\n",
       "      <th>36</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>(0, 5)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>(5, 0)</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>(5, 5)</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        example  row  col  coords  coords_idx\n",
       "worker         problem                                       \n",
       "A28AXX4NCWPH1F 36             0    0    5  (0, 5)           5\n",
       "               36             1    0    0  (0, 0)           0\n",
       "               36             2    5    0  (5, 0)          30\n",
       "               36             3    5    5  (5, 5)          35"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#temp_df[temp_df.index[0]]\n",
    "temp_df.loc[('A28AXX4NCWPH1F', 36)][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "0dcc03fe-6d0d-4eb1-b84b-cd0d4c8323cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>worker</th>\n",
       "      <th>example</th>\n",
       "      <th>row</th>\n",
       "      <th>col</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>problem</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>A28AXX4NCWPH1F</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A3HNEYFOIJWPH1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2587 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 worker  example  row  col\n",
       "problem                                   \n",
       "36       A28AXX4NCWPH1F        0    0    5\n",
       "36       A28AXX4NCWPH1F        1    0    0\n",
       "36       A28AXX4NCWPH1F        2    5    0\n",
       "36       A28AXX4NCWPH1F        3    5    5\n",
       "23       A28AXX4NCWPH1F        0    0    2\n",
       "...                 ...      ...  ...  ...\n",
       "21       A3HNEYFOIJWPH1        1    4    3\n",
       "0        A3HNEYFOIJWPH1        0    2    0\n",
       "0        A3HNEYFOIJWPH1        1    3    0\n",
       "0        A3HNEYFOIJWPH1        2    2    5\n",
       "0        A3HNEYFOIJWPH1        3    3    5\n",
       "\n",
       "[2587 rows x 4 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index('problem')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "850cb9ff-a5b6-48f6-b8e4-af74d5544f41",
   "metadata": {},
   "source": [
    "loop over unique problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ab824c5-219c-4a73-93f4-34bca0642a74",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}