ch07.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=a299071e366b4bd590198b85b8625f2b\n",
    "#%pushd book-materials\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "PREVIOUS_MAX_ROWS = pd.options.display.max_rows\n",
    "pd.options.display.max_rows = 25\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_colwidth = 82\n",
    "np.random.seed(12345)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc(\"figure\", figsize=(10, 6))\n",
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=835e9c4e6d3942638244064db595a5bd\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"shell_port\": 50369,\n",
      "  \"iopub_port\": 58941,\n",
      "  \"stdin_port\": 35813,\n",
      "  \"control_port\": 50907,\n",
      "  \"hb_port\": 45399,\n",
      "  \"ip\": \"127.0.0.1\",\n",
      "  \"key\": \"874524ca-ddc32af5cdf5b95285f5882f\",\n",
      "  \"transport\": \"tcp\",\n",
      "  \"signature_scheme\": \"hmac-sha256\",\n",
      "  \"kernel_name\": \"\"\n",
      "}\n",
      "\n",
      "Paste the above JSON into a file, and connect with:\n",
      "    $> jupyter <app> --existing <file>\n",
      "or, if you are local, you can connect with just:\n",
      "    $> jupyter <app> --existing kernel-a234a963-65f0-4a17-84f6-ba40351f3bdb.json\n",
      "or even just:\n",
      "    $> jupyter <app> --existing\n",
      "if this is the most recent Jupyter kernel you have started.\n"
     ]
    }
   ],
   "source": [
    "%connect_info\n",
    "#!jupyter qtconsole --style monokai --existing connect.json\n",
    "# Change your connect.json with %connect_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=290a88142afe4854bc724c7a6c735bae\n",
    "float_data = pd.Series([1.2, -3.5, np.nan, 0])\n",
    "float_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4b91ad05aea045cdbf13b5900cc1f73e\n",
    "float_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8996ec87330f486b96f59f1f394ccf6a\n",
    "string_data = pd.Series([\"aardvark\", np.nan, None, \"avocado\"])\n",
    "string_data\n",
    "string_data.isna()\n",
    "float_data = pd.Series([1, 2, None], dtype='float64')\n",
    "float_data\n",
    "float_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6dfb795c00c24da9a66cdd66870f9fa9\n",
    "data = pd.Series([1, np.nan, 3.5, np.nan, 7])\n",
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ce4c091e5f824312b06c6b1d6ff531fb\n",
    "data[data.notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=be32168b85724034b62a96f6fb6e7f2c\n",
    "data = pd.DataFrame([[1., 6.5, 3.], [1., np.nan, np.nan],\n",
    "                     [np.nan, np.nan, np.nan], [np.nan, 6.5, 3.]])\n",
    "data\n",
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=fc78d6354b4541f3a1c2416275e68c76\n",
    "data.dropna(how=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=87ba756b91eb4dd09d1b7e647a8ae7b9\n",
    "data[4] = np.nan\n",
    "data\n",
    "data.dropna(axis=\"columns\", how=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=93bc1cf6fc10429a924cea596353e388\n",
    "df = pd.DataFrame(np.random.standard_normal((7, 3)))\n",
    "df.iloc[:4, 1] = np.nan\n",
    "df.iloc[:2, 2] = np.nan\n",
    "df\n",
    "df.dropna()\n",
    "df.dropna(thresh=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ceee3ece49cc4169a1f8ac1e6954accd\n",
    "df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a03bb4bc12e24a3e941daa79068471ff\n",
    "df.fillna({1: 0.5, 2: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bf828716b9e04ef5a37200b5d5086b47\n",
    "df = pd.DataFrame(np.random.standard_normal((6, 3)))\n",
    "df.iloc[2:, 1] = np.nan\n",
    "df.iloc[4:, 2] = np.nan\n",
    "df\n",
    "df.fillna(method=\"ffill\")\n",
    "df.fillna(method=\"ffill\", limit=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=77b2b4714bd14dffb5e6b407fd0df952\n",
    "data = pd.Series([1., np.nan, 3.5, np.nan, 7])\n",
    "data.fillna(data.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=944377d16cdc466cb0a9e64fa1f79d5b\n",
    "data = pd.DataFrame({\"k1\": [\"one\", \"two\"] * 3 + [\"two\"],\n",
    "                     \"k2\": [1, 1, 2, 3, 3, 4, 4]})\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bd2994de11fd4012a835bd0ee2bcf4cb\n",
    "data.duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ac1c8d333c8d40068f33556cee5cb8b4\n",
    "data.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ac73e11080b04551bdf1a23729e75793\n",
    "data[\"v1\"] = range(7)\n",
    "data\n",
    "data.drop_duplicates(subset=[\"k1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=eb735c97c9b744aa923b8fb94ae22f64\n",
    "data.drop_duplicates([\"k1\", \"k2\"], keep=\"last\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4fd09f7a425741d5b1f61a28f2ec73a2\n",
    "data = pd.DataFrame({\"food\": [\"bacon\", \"pulled pork\", \"bacon\",\n",
    "                              \"pastrami\", \"corned beef\", \"bacon\",\n",
    "                              \"pastrami\", \"honey ham\", \"nova lox\"],\n",
    "                     \"ounces\": [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=1459d863e4e2488884aa21b5841fa88d\n",
    "meat_to_animal = {\n",
    "  \"bacon\": \"pig\",\n",
    "  \"pulled pork\": \"pig\",\n",
    "  \"pastrami\": \"cow\",\n",
    "  \"corned beef\": \"cow\",\n",
    "  \"honey ham\": \"pig\",\n",
    "  \"nova lox\": \"salmon\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a779fa7330de487fb907e801bf054696\n",
    "data[\"animal\"] = data[\"food\"].map(meat_to_animal)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4c24af18964744eda625a480f5cdd126\n",
    "def get_animal(x):\n",
    "    return meat_to_animal[x]\n",
    "data[\"food\"].map(get_animal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ec9f0e43f8da42ed8b3119331d09c853\n",
    "data = pd.Series([1., -999., 2., -999., -1000., 3.])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8fd47d92ab854f47bed89d8a72c9d8a0\n",
    "data.replace(-999, np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6bd3f8909caf4eed928f7164f277f8ef\n",
    "data.replace([-999, -1000], np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=467da543042d473aa9ed8b43e5daefec\n",
    "data.replace([-999, -1000], [np.nan, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e3dfba217f5a460d9bdf04ba573481ad\n",
    "data.replace({-999: np.nan, -1000: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=103a514cd451404fbc7b559bab5ea87c\n",
    "data = pd.DataFrame(np.arange(12).reshape((3, 4)),\n",
    "                    index=[\"Ohio\", \"Colorado\", \"New York\"],\n",
    "                    columns=[\"one\", \"two\", \"three\", \"four\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=aabdeea6a25f49a5a6949527b7d345bb\n",
    "def transform(x):\n",
    "    return x[:4].upper()\n",
    "\n",
    "data.index.map(transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8923ced04de44246b6d710395cc93348\n",
    "data.index = data.index.map(transform)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d125b3401a124c89a83007e36e3ee758\n",
    "data.rename(index=str.title, columns=str.upper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5f8e0d6ff0b4445d886c8ee0fb478191\n",
    "data.rename(index={\"OHIO\": \"INDIANA\"},\n",
    "            columns={\"three\": \"peekaboo\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5b066323cc4944c9938b36f4f8d52779\n",
    "ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=075935d1c34b41ba9ee1842e23605ad1\n",
    "bins = [18, 25, 35, 60, 100]\n",
    "age_categories = pd.cut(ages, bins)\n",
    "age_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ce9e07485ac641e291688784a05e841c\n",
    "age_categories.codes\n",
    "age_categories.categories\n",
    "age_categories.categories[0]\n",
    "pd.value_counts(age_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c40f7922de224c5682b37159bfb13adf\n",
    "pd.cut(ages, bins, right=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5d5d56eba8384d11bd5d38c378c9fbf1\n",
    "group_names = [\"Youth\", \"YoungAdult\", \"MiddleAged\", \"Senior\"]\n",
    "pd.cut(ages, bins, labels=group_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=33190ad9f1834874ba7cbd4066d8350b\n",
    "data = np.random.uniform(size=20)\n",
    "pd.cut(data, 4, precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5aa787199f2745d1a2358a097e9b23d9\n",
    "data = np.random.standard_normal(1000)\n",
    "quartiles = pd.qcut(data, 4, precision=2)\n",
    "quartiles\n",
    "pd.value_counts(quartiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6df8a2f012ad4735bbc92d478d946e1f\n",
    "pd.qcut(data, [0, 0.1, 0.5, 0.9, 1.]).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3979599fd2d247b7a32bf203a5c9a5c7\n",
    "data = pd.DataFrame(np.random.standard_normal((1000, 4)))\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=14662c0a588c488d83d792107f20e528\n",
    "col = data[2]\n",
    "col[col.abs() > 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2a1bb1c468cf40a58f3c5223a992b59b\n",
    "data[(data.abs() > 3).any(axis=\"columns\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=694246321bbd42adbb221b99486f388b\n",
    "data[data.abs() > 3] = np.sign(data) * 3\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d70c02173fe945abad1450a5709b520e\n",
    "np.sign(data).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=70d08ca5f48d48018126fd9ab1006314\n",
    "df = pd.DataFrame(np.arange(5 * 7).reshape((5, 7)))\n",
    "df\n",
    "sampler = np.random.permutation(5)\n",
    "sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f7c2008d514b4a549645c2c8a28861d9\n",
    "df.take(sampler)\n",
    "df.iloc[sampler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ffbccf111b60408395f9eb116f6fc39a\n",
    "column_sampler = np.random.permutation(7)\n",
    "column_sampler\n",
    "df.take(column_sampler, axis=\"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c2429f2bce484acca0f590eed87cb896\n",
    "df.sample(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f98970c0ba4942d880cf0d4076960bc3\n",
    "choices = pd.Series([5, 7, -1, 6, 4])\n",
    "choices.sample(n=10, replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=590105f5b54b4b798b9c925339f1c000\n",
    "df = pd.DataFrame({\"key\": [\"b\", \"b\", \"a\", \"c\", \"a\", \"b\"],\n",
    "                   \"data1\": range(6)})\n",
    "df\n",
    "pd.get_dummies(df[\"key\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c9bf047cafa2489cb537d963eb18d16a\n",
    "dummies = pd.get_dummies(df[\"key\"], prefix=\"key\")\n",
    "df_with_dummy = df[[\"data1\"]].join(dummies)\n",
    "df_with_dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython allow_exceptions id=e311a84ee00c40479411ed79fc5a6c69\n",
    "mnames = [\"movie_id\", \"title\", \"genres\"]\n",
    "movies = pd.read_table(\"datasets/movielens/movies.dat\", sep=\"::\",\n",
    "                       header=None, names=mnames, engine=\"python\")\n",
    "movies[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8e085399c22341ee9632bf5efcb4a6dc\n",
    "dummies = movies[\"genres\"].str.get_dummies(\"|\")\n",
    "dummies.iloc[:10, :6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9b2a0b211d424f6798485b72bd572e08\n",
    "movies_windic = movies.join(dummies.add_prefix(\"Genre_\"))\n",
    "movies_windic.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f3df965a3e024e918c1e67e647a6589a\n",
    "np.random.seed(12345) # to make the example repeatable\n",
    "values = np.random.uniform(size=10)\n",
    "values\n",
    "bins = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
    "pd.get_dummies(pd.cut(values, bins))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=10771102e8084f5fa4a16200a93d058d\n",
    "s = pd.Series([1, 2, 3, None])\n",
    "s\n",
    "s.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=66923eb117b34b21b548507e349d387d\n",
    "s = pd.Series([1, 2, 3, None], dtype=pd.Int64Dtype())\n",
    "s\n",
    "s.isna()\n",
    "s.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8a80094d8808433599836ea7197e55b9\n",
    "s[3]\n",
    "s[3] is pd.NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=43b53a40545c451c94728b65b40c574e\n",
    "s = pd.Series([1, 2, 3, None], dtype=\"Int64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=85af440e08ae45d9b570c8b65f496567\n",
    "s = pd.Series(['one', 'two', None, 'three'], dtype=pd.StringDtype())\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=81a4d68317534367bdbd458b98a55329\n",
    "df = pd.DataFrame({\"A\": [1, 2, None, 4],\n",
    "                   \"B\": [\"one\", \"two\", \"three\", None],\n",
    "                   \"C\": [False, None, False, True]})\n",
    "df\n",
    "df[\"A\"] = df[\"A\"].astype(\"Int64\")\n",
    "df[\"B\"] = df[\"B\"].astype(\"string\")\n",
    "df[\"C\"] = df[\"C\"].astype(\"boolean\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=fe9efc6724224b7b9eccd32750524def\n",
    "val = \"a,b,  guido\"\n",
    "val.split(\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9724ab6a052943e7803c2f855b47b29f\n",
    "pieces = [x.strip() for x in val.split(\",\")]\n",
    "pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e80b9a9828014622b86c088759d5e5a0\n",
    "first, second, third = pieces\n",
    "first + \"::\" + second + \"::\" + third"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=06baf1c7f132476f90f73524278f5972\n",
    "\"::\".join(pieces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d5571ea9402c4182af832d4b0b659ad4\n",
    "\"guido\" in val\n",
    "val.index(\",\")\n",
    "val.find(\":\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython allow_exceptions id=ee7ecc7978534b3883cf050a76245ba3\n",
    "val.index(\":\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6c3ac8963e0348d5900d3b21f3d61421\n",
    "val.count(\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2385159093a248abbb0b590843359737\n",
    "val.replace(\",\", \"::\")\n",
    "val.replace(\",\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b430b6ee3c904fa38a207e995ad4e152\n",
    "import re\n",
    "text = \"foo    bar\\t baz  \\tqux\"\n",
    "re.split(r\"\\s+\", text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2af33f0f709d42a2853daaabafede024\n",
    "regex = re.compile(r\"\\s+\")\n",
    "regex.split(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=746b726a56dd4375a8a459b74d65789c\n",
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=8f2d09408b41491aa269044fc4375d93\n",
    "text = \"\"\"Dave dave@google.com\n",
    "Steve steve@gmail.com\n",
    "Rob rob@gmail.com\n",
    "Ryan ryan@yahoo.com\"\"\"\n",
    "pattern = r\"[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\"\n",
    "\n",
    "# re.IGNORECASE makes the regex case insensitive\n",
    "regex = re.compile(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4efa726d3118445db3dfa63e69147a3e\n",
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5d3ea127f5064a0eabafa8f07982c0fe\n",
    "m = regex.search(text)\n",
    "m\n",
    "text[m.start():m.end()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=56352eb1b66041658dc4a2df7cdd3659\n",
    "print(regex.match(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=16b2b19e1790421f8cbfd96272f49964\n",
    "print(regex.sub(\"REDACTED\", text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=48a7da4e764149fc9fbbc73d8836e703\n",
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\"\n",
    "regex = re.compile(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0e0269c4dfb748a787340d99bd51af16\n",
    "m = regex.match(\"wesm@bright.net\")\n",
    "m.groups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ba6371f22b31476d99904236e025caf2\n",
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=adf56cb7344246a4981c9ccbc4520e45\n",
    "print(regex.sub(r\"Username: \\1, Domain: \\2, Suffix: \\3\", text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d2dae337d0724a7cabb69f0d2e45d21a\n",
    "data = {\"Dave\": \"dave@google.com\", \"Steve\": \"steve@gmail.com\",\n",
    "        \"Rob\": \"rob@gmail.com\", \"Wes\": np.nan}\n",
    "data = pd.Series(data)\n",
    "data\n",
    "data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b1a32c08159845ad97915762d2e9104c\n",
    "data.str.contains(\"gmail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e83b7f3cb11b42ea8344da39bc8313ed\n",
    "data_as_string_ext = data.astype('string')\n",
    "data_as_string_ext\n",
    "data_as_string_ext.str.contains(\"gmail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=abd3d1bd7239406d8d764f812cb04018\n",
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\"\n",
    "data.str.findall(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=07c02acaf03844578f9b18af036e96b4\n",
    "matches = data.str.findall(pattern, flags=re.IGNORECASE).str[0]\n",
    "matches\n",
    "matches.str.get(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=140f72f7ebdf44fc832ac76596f1f500\n",
    "data.str[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=086c74fc48364b859ab30d4c9278e96d\n",
    "data.str.extract(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5e0d1d0145d64d80870482f06001910e\n",
    "values = pd.Series(['apple', 'orange', 'apple',\n",
    "                    'apple'] * 2)\n",
    "values\n",
    "pd.unique(values)\n",
    "pd.value_counts(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6b72957b5a1c4882828b5b6fc55bc645\n",
    "values = pd.Series([0, 1, 0, 0] * 2)\n",
    "dim = pd.Series(['apple', 'orange'])\n",
    "values\n",
    "dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c4c6450bbb7f4c74b579d4cc989d0547\n",
    "dim.take(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=922c7a3dfaf5478c8ece876bf6e8c8a4\n",
    "fruits = ['apple', 'orange', 'apple', 'apple'] * 2\n",
    "N = len(fruits)\n",
    "rng = np.random.default_rng(seed=12345)\n",
    "df = pd.DataFrame({'fruit': fruits,\n",
    "                   'basket_id': np.arange(N),\n",
    "                   'count': rng.integers(3, 15, size=N),\n",
    "                   'weight': rng.uniform(0, 4, size=N)},\n",
    "                  columns=['basket_id', 'fruit', 'count', 'weight'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d3efa50a15b64f78b8ee5d1bffd9e3db\n",
    "fruit_cat = df['fruit'].astype('category')\n",
    "fruit_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5b3590609c7f4a2984add7aefc62727c\n",
    "c = fruit_cat.array\n",
    "type(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4d5fd7e39e7b4dc4bf59a7597a66cbe1\n",
    "c.categories\n",
    "c.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a5d2afb03a3d46fe99b79633b5480fb7\n",
    "dict(enumerate(c.categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b4e849ae87304ac1a2b717ecafbf58c4\n",
    "df['fruit'] = df['fruit'].astype('category')\n",
    "df[\"fruit\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=adba8c41d0094e49943f4c0f03bfb560\n",
    "my_categories = pd.Categorical(['foo', 'bar', 'baz', 'foo', 'bar'])\n",
    "my_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=dbb5b2424ba146f0b001398542f64ae3\n",
    "categories = ['foo', 'bar', 'baz']\n",
    "codes = [0, 1, 2, 0, 0, 1]\n",
    "my_cats_2 = pd.Categorical.from_codes(codes, categories)\n",
    "my_cats_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2ee990370ede41ddbcf285a6c1330ee0\n",
    "ordered_cat = pd.Categorical.from_codes(codes, categories,\n",
    "                                        ordered=True)\n",
    "ordered_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=27e720c7999143b8bc6fa95797bb78fd\n",
    "my_cats_2.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3268082ea833473888cb6df910208de8\n",
    "rng = np.random.default_rng(seed=12345)\n",
    "draws = rng.standard_normal(1000)\n",
    "draws[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6d7daaf08a8240dea51848a3a6cc211d\n",
    "bins = pd.qcut(draws, 4)\n",
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6faabc704b9c436fa0e32ac6755e6359\n",
    "bins = pd.qcut(draws, 4, labels=['Q1', 'Q2', 'Q3', 'Q4'])\n",
    "bins\n",
    "bins.codes[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0f2080e060dd4d848ff1a9d846c8a71e\n",
    "bins = pd.Series(bins, name='quartile')\n",
    "results = (pd.Series(draws)\n",
    "           .groupby(bins)\n",
    "           .agg(['count', 'min', 'max'])\n",
    "           .reset_index())\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4d1f92dfe8e34a6d8173c0aca18d8b55\n",
    "results['quartile']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=cb512af2f6bf4eb5b312ae330bd4c91e\n",
    "N = 10_000_000\n",
    "labels = pd.Series(['foo', 'bar', 'baz', 'qux'] * (N // 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2978248572a743be8529a1668d424e8a\n",
    "categories = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a16f1610329347ffa2f28661941350b7\n",
    "labels.memory_usage(deep=True)\n",
    "categories.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=dd7310d96d4d4c24bb7e3e9ad26b066f\n",
    "%time _ = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=aca60c356d264f7c872287f1bfdf75c9\n",
    "%timeit labels.value_counts()\n",
    "%timeit categories.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9dd1c0395c0743db8a283da215ae7f50\n",
    "s = pd.Series(['a', 'b', 'c', 'd'] * 2)\n",
    "cat_s = s.astype('category')\n",
    "cat_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=84308427acbc491381531a16cd3bcc15\n",
    "cat_s.cat.codes\n",
    "cat_s.cat.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7b07ac0b91ef42809b1183ccc41555c1\n",
    "actual_categories = ['a', 'b', 'c', 'd', 'e']\n",
    "cat_s2 = cat_s.cat.set_categories(actual_categories)\n",
    "cat_s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=919709709a4c4c8d93e135d137ea0757\n",
    "cat_s.value_counts()\n",
    "cat_s2.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0350789cd1fe41c6908b5e604ea34eb0\n",
    "cat_s3 = cat_s[cat_s.isin(['a', 'b'])]\n",
    "cat_s3\n",
    "cat_s3.cat.remove_unused_categories()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c6b7347f928a4580b5dac67f798d9e28\n",
    "cat_s = pd.Series(['a', 'b', 'c', 'd'] * 2, dtype='category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=95950502e1bc4cfba88e7891c8062ee6\n",
    "pd.get_dummies(cat_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=b8746f91160c4c9da000aeb4891514bf\n",
    "%popd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=621a2c4436a44e5db40091b1e4f8a167\n",
    "pd.options.display.max_rows = PREVIOUS_MAX_ROWS"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}