33-exporting-code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/matt/envs/menv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3418: DtypeWarning: Columns (7) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "url = 'https://github.com/mattharrison/datasets/raw/master'\\\n",
    "      '/data/dirtydevil.txt'\n",
    "df = pd.read_csv(url, skiprows=lambda num: num <34 or num == 35,\n",
    "                 sep='\\t')\n",
    "def to_denver_time(df_, time_col, tz_col):\n",
    "    return (df_\n",
    "       .assign(**{tz_col: df_[tz_col].replace('MDT', 'MST7MDT')})\n",
    "       .groupby(tz_col)\n",
    "       [time_col]\n",
    "       .transform(lambda s: pd.to_datetime(s)\n",
    "           .dt.tz_localize(s.name, ambiguous=True)\n",
    "           .dt.tz_convert('America/Denver'))\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_river(df_):\n",
    "    return (df_\n",
    "     .assign(datetime=to_denver_time(df_, 'datetime', 'tz_cd'))\n",
    "     .rename(columns={'144166_00060': 'cfs',\n",
    "                      '144167_00065': 'gage_height'})\n",
    "     .set_index('datetime')\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:15:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:45:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 02:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 08:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.53</td>\n",
       "      <td>P</td>\n",
       "      <td>6.16</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 08:45:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:15:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>539305 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          agency_cd  site_no tz_cd    cfs 144166_00060_cd  \\\n",
       "datetime                                                                    \n",
       "2001-05-07 01:00:00-06:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:15:00-06:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:30:00-06:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:45:00-06:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "2001-05-07 02:00:00-06:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "...                             ...      ...   ...    ...             ...   \n",
       "2020-09-28 08:30:00-06:00      USGS  9333500   MDT   9.53               P   \n",
       "2020-09-28 08:45:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:00:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:15:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:30:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "\n",
       "                           gage_height 144167_00065_cd  \n",
       "datetime                                                \n",
       "2001-05-07 01:00:00-06:00          NaN             NaN  \n",
       "2001-05-07 01:15:00-06:00          NaN             NaN  \n",
       "2001-05-07 01:30:00-06:00          NaN             NaN  \n",
       "2001-05-07 01:45:00-06:00          NaN             NaN  \n",
       "2001-05-07 02:00:00-06:00          NaN             NaN  \n",
       "...                                ...             ...  \n",
       "2020-09-28 08:30:00-06:00         6.16               P  \n",
       "2020-09-28 08:45:00-06:00         6.15               P  \n",
       "2020-09-28 09:00:00-06:00         6.15               P  \n",
       "2020-09-28 09:15:00-06:00         6.15               P  \n",
       "2020-09-28 09:30:00-06:00         6.15               P  \n",
       "\n",
       "[539305 rows x 7 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd = tweak_river(df)\n",
    "dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "dd.to_csv('/tmp/dd.csv')  # doctest: +SKIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datetime,agency_cd,site_no,tz_cd,cfs,144166_00060_cd,gage_height,144167_00065_cd\n",
      "2001-05-07 01:00:00-06:00,USGS,9333500,MDT,71.0,A:[91],,\n",
      "2001-05-07 01:15:00-06:00,USGS,9333500,MDT,71.0,A:[91],,\n",
      "2001-05-07 01:30:00-06:00,USGS,9333500,MDT,71.0,A:[91],,\n",
      "2001-05-07 01:45:00-06:00,USGS,9333500,MDT,70.0,A:[91],,\n",
      "2001-05-07 02:00:00-06:00,USGS,9333500,MDT,70.0,A:[91],,\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(dd.head(5).to_csv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/matt/envs/menv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3418: DtypeWarning: Columns (7) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "dd2 = pd.read_csv('/tmp/dd.csv', index_col='datetime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Excel does not support datetimes with timezones. Please ensure that datetimes are timezone unaware before writing to Excel.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-0ac6c2e2e196>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_excel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/tmp/dd.xlsx'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mto_excel\u001b[0;34m(self, excel_writer, sheet_name, na_rep, float_format, columns, header, index, index_label, startrow, startcol, engine, merge_cells, encoding, inf_rep, verbose, freeze_panes, storage_options)\u001b[0m\n\u001b[1;32m   2282\u001b[0m             \u001b[0minf_rep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minf_rep\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2283\u001b[0m         )\n\u001b[0;32m-> 2284\u001b[0;31m         formatter.write(\n\u001b[0m\u001b[1;32m   2285\u001b[0m             \u001b[0mexcel_writer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2286\u001b[0m             \u001b[0msheet_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msheet_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/io/formats/excel.py\u001b[0m in \u001b[0;36mwrite\u001b[0;34m(self, writer, sheet_name, startrow, startcol, freeze_panes, engine, storage_options)\u001b[0m\n\u001b[1;32m    838\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    839\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 840\u001b[0;31m             writer.write_cells(\n\u001b[0m\u001b[1;32m    841\u001b[0m                 \u001b[0mformatted_cells\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m                 \u001b[0msheet_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/io/excel/_xlsxwriter.py\u001b[0m in \u001b[0;36mwrite_cells\u001b[0;34m(self, cells, sheet_name, startrow, startcol, freeze_panes)\u001b[0m\n\u001b[1;32m    225\u001b[0m             \u001b[0mwks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfreeze_panes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfreeze_panes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mcell\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcells\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m             \u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfmt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_with_fmt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/io/formats/excel.py\u001b[0m in \u001b[0;36mget_formatted_cells\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    776\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_formatted_cells\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mIterable\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mExcelCell\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    777\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcell\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_format_header\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_format_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 778\u001b[0;31m             \u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_format_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    779\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    780\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/io/formats/excel.py\u001b[0m in \u001b[0;36m_format_value\u001b[0;34m(self, val)\u001b[0m\n\u001b[1;32m    524\u001b[0m                 \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat_format\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"tzinfo\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 526\u001b[0;31m             raise ValueError(\n\u001b[0m\u001b[1;32m    527\u001b[0m                 \u001b[0;34m\"Excel does not support datetimes with \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m                 \u001b[0;34m\"timezones. Please ensure that datetimes \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Excel does not support datetimes with timezones. Please ensure that datetimes are timezone unaware before writing to Excel."
     ]
    }
   ],
   "source": [
    "dd.to_excel('/tmp/dd.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "(dd  # doctest: +SKIP\n",
    " .reset_index()\n",
    " .assign(datetime=lambda df_: df_.datetime.dt.tz_convert(tz=None))\n",
    " .set_index('datetime')\n",
    " .to_excel('/tmp/dd.xlsx')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter('/tmp/dd2.xlsx')\n",
    "dd2 = (dd    \n",
    " .reset_index()\n",
    " .assign(datetime=lambda df_: df_.datetime.dt.tz_convert(tz=None))\n",
    " .set_index('datetime')\n",
    ")\n",
    "(dd2      # doctest: +SKIP\n",
    " .loc['2010':'2010-12-31']\n",
    " .to_excel(writer, sheet_name='2010')\n",
    ")\n",
    "(dd2     # doctest: +SKIP\n",
    " .loc['2011':'2011-12-31'] \n",
    " .to_excel(writer, sheet_name='2011')\n",
    ")\n",
    "writer.save()      # doctest: +SKIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "lines_to_next_cell": 2,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "feather does not support serializing <class 'pandas.core.indexes.datetimes.DatetimeIndex'> for the index; you can .reset_index() to make the index into column(s)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-73127889b6d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_feather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/tmp/dd.fea'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    205\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 207\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mto_feather\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m   2517\u001b[0m         \u001b[0;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeather_format\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_feather\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2519\u001b[0;31m         \u001b[0mto_feather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2520\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2521\u001b[0m     @doc(\n",
      "\u001b[0;32m~/envs/menv/lib/python3.8/site-packages/pandas/io/feather_format.py\u001b[0m in \u001b[0;36mto_feather\u001b[0;34m(df, path, storage_options, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mInt64Index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRangeIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mtyp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         raise ValueError(\n\u001b[0m\u001b[1;32m     62\u001b[0m             \u001b[0;34mf\"feather does not support serializing {typ} \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m             \u001b[0;34m\"for the index; you can .reset_index() to make the index into column(s)\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: feather does not support serializing <class 'pandas.core.indexes.datetimes.DatetimeIndex'> for the index; you can .reset_index() to make the index into column(s)"
     ]
    }
   ],
   "source": [
    "dd.to_feather('/tmp/dd.fea')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "(dd\n",
    " .reset_index()\n",
    " .to_feather('/tmp/dd.fea')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd2 = pd.read_feather('/tmp/dd.fea')\n",
    "dd2.set_index('datetime').equals(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "con = sqlite3.connect('dd.db')\n",
    "dd.to_sql('dd', con, if_exists='replace')  # doctest: +SKIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sqlalchemy as sa\n",
    "eng = sa.create_engine('sqlite:///dd.db')\n",
    "sa_con = eng.connect()\n",
    "dd2 = pd.read_sql('dd', sa_con, index_col='datetime')  # doctest: +SKIP\n",
    "dd2.equals(dd)  # doctest: +SKIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:00:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:15:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:30:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:45:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 02:00:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 08:30:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.53</td>\n",
       "      <td>P</td>\n",
       "      <td>6.16</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 08:45:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:00:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:15:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:30:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>539305 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    agency_cd  site_no tz_cd    cfs 144166_00060_cd  \\\n",
       "datetime                                                              \n",
       "2001-05-07 01:00:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:15:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:30:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:45:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "2001-05-07 02:00:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "...                       ...      ...   ...    ...             ...   \n",
       "2020-09-28 08:30:00      USGS  9333500   MDT   9.53               P   \n",
       "2020-09-28 08:45:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:00:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:15:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:30:00      USGS  9333500   MDT   9.20               P   \n",
       "\n",
       "                     gage_height 144167_00065_cd  \n",
       "datetime                                          \n",
       "2001-05-07 01:00:00          NaN            None  \n",
       "2001-05-07 01:15:00          NaN            None  \n",
       "2001-05-07 01:30:00          NaN            None  \n",
       "2001-05-07 01:45:00          NaN            None  \n",
       "2001-05-07 02:00:00          NaN            None  \n",
       "...                          ...             ...  \n",
       "2020-09-28 08:30:00         6.16               P  \n",
       "2020-09-28 08:45:00         6.15               P  \n",
       "2020-09-28 09:00:00         6.15               P  \n",
       "2020-09-28 09:15:00         6.15               P  \n",
       "2020-09-28 09:30:00         6.15               P  \n",
       "\n",
       "[539305 rows x 7 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dd2  # doctest: +SKIP\n",
    " .reset_index()\n",
    " .assign(datetime=lambda df_: df_.datetime\n",
    "       .dt.tz_localize('America/Denver', ambiguous=False))\n",
    " .set_index('datetime')\n",
    " .equals(dd)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "obj = dd.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd2 = pd.DataFrame.from_dict(obj)\n",
    "dd.equals(dd2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2001-05-07 07:00:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 07:15:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 07:30:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 07:45:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 08:00:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 14:30:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.53</td>\n",
       "      <td>P</td>\n",
       "      <td>6.16</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 14:45:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 15:00:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 15:15:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 15:30:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>539305 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    agency_cd  site_no tz_cd    cfs 144166_00060_cd  \\\n",
       "2001-05-07 07:00:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 07:15:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 07:30:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 07:45:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "2001-05-07 08:00:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "...                       ...      ...   ...    ...             ...   \n",
       "2020-09-28 14:30:00      USGS  9333500   MDT   9.53               P   \n",
       "2020-09-28 14:45:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 15:00:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 15:15:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 15:30:00      USGS  9333500   MDT   9.20               P   \n",
       "\n",
       "                     gage_height 144167_00065_cd  \n",
       "2001-05-07 07:00:00          NaN            None  \n",
       "2001-05-07 07:15:00          NaN            None  \n",
       "2001-05-07 07:30:00          NaN            None  \n",
       "2001-05-07 07:45:00          NaN            None  \n",
       "2001-05-07 08:00:00          NaN            None  \n",
       "...                          ...             ...  \n",
       "2020-09-28 14:30:00         6.16               P  \n",
       "2020-09-28 14:45:00         6.15               P  \n",
       "2020-09-28 15:00:00         6.15               P  \n",
       "2020-09-28 15:15:00         6.15               P  \n",
       "2020-09-28 15:30:00         6.15               P  \n",
       "\n",
       "[539305 rows x 7 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.to_json('/tmp/dd.json.gz')\n",
    "dd2 = pd.read_json('/tmp/dd.json')\n",
    "dd2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd2.equals(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "dd3 = (dd2\n",
    " .reset_index()\n",
    " .rename(columns={'index':'datetime'})\n",
    " .assign(datetime=lambda df_: df_.datetime.dt.tz_localize(tz='UTC')\n",
    "         .dt.tz_convert('America/Denver'))\n",
    " .set_index('datetime')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:15:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:45:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 02:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.00</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 08:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.53</td>\n",
       "      <td>P</td>\n",
       "      <td>6.16</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 08:45:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:15:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-28 09:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>9.20</td>\n",
       "      <td>P</td>\n",
       "      <td>6.15</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>539305 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          agency_cd  site_no tz_cd    cfs 144166_00060_cd  \\\n",
       "datetime                                                                    \n",
       "2001-05-07 01:00:00-06:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:15:00-06:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:30:00-06:00      USGS  9333500   MDT  71.00          A:[91]   \n",
       "2001-05-07 01:45:00-06:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "2001-05-07 02:00:00-06:00      USGS  9333500   MDT  70.00          A:[91]   \n",
       "...                             ...      ...   ...    ...             ...   \n",
       "2020-09-28 08:30:00-06:00      USGS  9333500   MDT   9.53               P   \n",
       "2020-09-28 08:45:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:00:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:15:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "2020-09-28 09:30:00-06:00      USGS  9333500   MDT   9.20               P   \n",
       "\n",
       "                           gage_height 144167_00065_cd  \n",
       "datetime                                                \n",
       "2001-05-07 01:00:00-06:00          NaN            None  \n",
       "2001-05-07 01:15:00-06:00          NaN            None  \n",
       "2001-05-07 01:30:00-06:00          NaN            None  \n",
       "2001-05-07 01:45:00-06:00          NaN            None  \n",
       "2001-05-07 02:00:00-06:00          NaN            None  \n",
       "...                                ...             ...  \n",
       "2020-09-28 08:30:00-06:00         6.16               P  \n",
       "2020-09-28 08:45:00-06:00         6.15               P  \n",
       "2020-09-28 09:00:00-06:00         6.15               P  \n",
       "2020-09-28 09:15:00-06:00         6.15               P  \n",
       "2020-09-28 09:30:00-06:00         6.15               P  \n",
       "\n",
       "[539305 rows x 7 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd3.equals(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd3.round(3).equals(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agency_cd</th>\n",
       "      <th>site_no</th>\n",
       "      <th>tz_cd</th>\n",
       "      <th>cfs</th>\n",
       "      <th>144166_00060_cd</th>\n",
       "      <th>gage_height</th>\n",
       "      <th>144167_00065_cd</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.0</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:15:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.0</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:30:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>71.0</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 01:45:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.0</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001-05-07 02:00:00-06:00</th>\n",
       "      <td>USGS</td>\n",
       "      <td>9333500</td>\n",
       "      <td>MDT</td>\n",
       "      <td>70.0</td>\n",
       "      <td>A:[91]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          agency_cd  site_no tz_cd   cfs 144166_00060_cd  \\\n",
       "datetime                                                                   \n",
       "2001-05-07 01:00:00-06:00      USGS  9333500   MDT  71.0          A:[91]   \n",
       "2001-05-07 01:15:00-06:00      USGS  9333500   MDT  71.0          A:[91]   \n",
       "2001-05-07 01:30:00-06:00      USGS  9333500   MDT  71.0          A:[91]   \n",
       "2001-05-07 01:45:00-06:00      USGS  9333500   MDT  70.0          A:[91]   \n",
       "2001-05-07 02:00:00-06:00      USGS  9333500   MDT  70.0          A:[91]   \n",
       "\n",
       "                           gage_height 144167_00065_cd  \n",
       "datetime                                                \n",
       "2001-05-07 01:00:00-06:00          NaN             NaN  \n",
       "2001-05-07 01:15:00-06:00          NaN             NaN  \n",
       "2001-05-07 01:30:00-06:00          NaN             NaN  \n",
       "2001-05-07 01:45:00-06:00          NaN             NaN  \n",
       "2001-05-07 02:00:00-06:00          NaN             NaN  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"agency_cd\":{\"989218800000\":\"USGS\",\"989219700000\":\"USGS\",\"9'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.head().to_json()[:60]"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}