Pandas Basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c2700a15",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6405a650",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "      <th>col4</th>\n",
       "      <th>col5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Row1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Row2</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Row3</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Row4</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      col1  col2  col3  col4  col5\n",
       "Row1     0     1     2     3     4\n",
       "Row2     5     6     7     8     9\n",
       "Row3    10    11    12    13    14\n",
       "Row4    15    16    17    18    19"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.arange(0,20).reshape(4,5),index=['Row1','Row2','Row3','Row4'],\n",
    "                  columns=['col1','col2','col3','col4','col5'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b92d443d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5201772d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "col1    0\n",
       "col2    1\n",
       "col3    2\n",
       "col4    3\n",
       "col5    4\n",
       "Name: Row1, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Accessing the elements\n",
    "df.loc['Row1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f3fedf58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df.loc['Row1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ec88dd9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## df.iloc[rowindex(can be sliced index),colindex(can be sliced index)]\n",
    "df.iloc[1,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "84d08a35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      col3  col4  col5\n",
      "Row2     7     8     9\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.iloc[1:2,2:])\n",
    "type(df.iloc[0:2,0:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "94ea465b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n",
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "## If you have more than 1 column and 1 row, then your type is dataframe else the type is series\n",
    "print(type(df.iloc[0:2,0])) ## It has 2 rows and 0 column\n",
    "print(type(df.iloc[0:2,0:1])) ## It has 2 rows and 1 column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ddd73a7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  2,  3,  4],\n",
       "       [ 6,  7,  8,  9],\n",
       "       [11, 12, 13, 14],\n",
       "       [16, 17, 18, 19]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Convert dataframe into arrays\n",
    "df.iloc[:,1:].values ## This will remove the column names and row names and convert it to array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4f9882dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "col1    0\n",
       "col2    0\n",
       "col3    0\n",
       "col4    0\n",
       "col5    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## This just gives the number of null values in each column\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43a247db",
   "metadata": {},
   "source": [
    "## Read JSON, Read HTML, Read PICKLE, Read EXCEL Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "45fc0d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "568c29c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "      <th>col4</th>\n",
       "      <th>col5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Row1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Row2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Row3</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Row4</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Unnamed: 0  col1  col2  col3  col4  col5\n",
       "0       Row1     0     1     2     3     4\n",
       "1       Row2     5     6     7     8     9\n",
       "2       Row3    10    11    12    13    14\n",
       "3       Row4    15    16    17    18    19"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8af7276a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('mercedesbenz.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d823c5f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>y</th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X8</th>\n",
       "      <th>...</th>\n",
       "      <th>X375</th>\n",
       "      <th>X376</th>\n",
       "      <th>X377</th>\n",
       "      <th>X378</th>\n",
       "      <th>X379</th>\n",
       "      <th>X380</th>\n",
       "      <th>X382</th>\n",
       "      <th>X383</th>\n",
       "      <th>X384</th>\n",
       "      <th>X385</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>130.81</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>a</td>\n",
       "      <td>d</td>\n",
       "      <td>u</td>\n",
       "      <td>j</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>88.53</td>\n",
       "      <td>k</td>\n",
       "      <td>t</td>\n",
       "      <td>av</td>\n",
       "      <td>e</td>\n",
       "      <td>d</td>\n",
       "      <td>y</td>\n",
       "      <td>l</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>76.26</td>\n",
       "      <td>az</td>\n",
       "      <td>w</td>\n",
       "      <td>n</td>\n",
       "      <td>c</td>\n",
       "      <td>d</td>\n",
       "      <td>x</td>\n",
       "      <td>j</td>\n",
       "      <td>x</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>80.62</td>\n",
       "      <td>az</td>\n",
       "      <td>t</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>d</td>\n",
       "      <td>x</td>\n",
       "      <td>l</td>\n",
       "      <td>e</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>78.02</td>\n",
       "      <td>az</td>\n",
       "      <td>v</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>d</td>\n",
       "      <td>h</td>\n",
       "      <td>d</td>\n",
       "      <td>n</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 378 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       y  X0 X1  X2 X3 X4 X5 X6 X8  ...  X375  X376  X377  X378  X379  \\\n",
       "0   0  130.81   k  v  at  a  d  u  j  o  ...     0     0     1     0     0   \n",
       "1   6   88.53   k  t  av  e  d  y  l  o  ...     1     0     0     0     0   \n",
       "2   7   76.26  az  w   n  c  d  x  j  x  ...     0     0     0     0     0   \n",
       "3   9   80.62  az  t   n  f  d  x  l  e  ...     0     0     0     0     0   \n",
       "4  13   78.02  az  v   n  f  d  h  d  n  ...     0     0     0     0     0   \n",
       "\n",
       "   X380  X382  X383  X384  X385  \n",
       "0     0     0     0     0     0  \n",
       "1     0     0     0     0     0  \n",
       "2     0     1     0     0     0  \n",
       "3     0     0     0     0     0  \n",
       "4     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 378 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "88336b91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4209 entries, 0 to 4208\n",
      "Columns: 378 entries, ID to X385\n",
      "dtypes: float64(1), int64(369), object(8)\n",
      "memory usage: 12.1+ MB\n"
     ]
    }
   ],
   "source": [
    "## Gives the detailed info about the dataframe (#rows, #columns, memory usage, \n",
    "## types used in the dataframe)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c2daa31c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>y</th>\n",
       "      <th>X10</th>\n",
       "      <th>X11</th>\n",
       "      <th>X12</th>\n",
       "      <th>X13</th>\n",
       "      <th>X14</th>\n",
       "      <th>X15</th>\n",
       "      <th>X16</th>\n",
       "      <th>X17</th>\n",
       "      <th>...</th>\n",
       "      <th>X375</th>\n",
       "      <th>X376</th>\n",
       "      <th>X377</th>\n",
       "      <th>X378</th>\n",
       "      <th>X379</th>\n",
       "      <th>X380</th>\n",
       "      <th>X382</th>\n",
       "      <th>X383</th>\n",
       "      <th>X384</th>\n",
       "      <th>X385</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.0</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "      <td>4209.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4205.960798</td>\n",
       "      <td>100.669318</td>\n",
       "      <td>0.013305</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.075077</td>\n",
       "      <td>0.057971</td>\n",
       "      <td>0.428130</td>\n",
       "      <td>0.000475</td>\n",
       "      <td>0.002613</td>\n",
       "      <td>0.007603</td>\n",
       "      <td>...</td>\n",
       "      <td>0.318841</td>\n",
       "      <td>0.057258</td>\n",
       "      <td>0.314802</td>\n",
       "      <td>0.020670</td>\n",
       "      <td>0.009503</td>\n",
       "      <td>0.008078</td>\n",
       "      <td>0.007603</td>\n",
       "      <td>0.001663</td>\n",
       "      <td>0.000475</td>\n",
       "      <td>0.001426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2437.608688</td>\n",
       "      <td>12.679381</td>\n",
       "      <td>0.114590</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.263547</td>\n",
       "      <td>0.233716</td>\n",
       "      <td>0.494867</td>\n",
       "      <td>0.021796</td>\n",
       "      <td>0.051061</td>\n",
       "      <td>0.086872</td>\n",
       "      <td>...</td>\n",
       "      <td>0.466082</td>\n",
       "      <td>0.232363</td>\n",
       "      <td>0.464492</td>\n",
       "      <td>0.142294</td>\n",
       "      <td>0.097033</td>\n",
       "      <td>0.089524</td>\n",
       "      <td>0.086872</td>\n",
       "      <td>0.040752</td>\n",
       "      <td>0.021796</td>\n",
       "      <td>0.037734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>72.110000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2095.000000</td>\n",
       "      <td>90.820000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4220.000000</td>\n",
       "      <td>99.150000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6314.000000</td>\n",
       "      <td>109.010000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8417.000000</td>\n",
       "      <td>265.320000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 370 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                ID            y          X10     X11          X12  \\\n",
       "count  4209.000000  4209.000000  4209.000000  4209.0  4209.000000   \n",
       "mean   4205.960798   100.669318     0.013305     0.0     0.075077   \n",
       "std    2437.608688    12.679381     0.114590     0.0     0.263547   \n",
       "min       0.000000    72.110000     0.000000     0.0     0.000000   \n",
       "25%    2095.000000    90.820000     0.000000     0.0     0.000000   \n",
       "50%    4220.000000    99.150000     0.000000     0.0     0.000000   \n",
       "75%    6314.000000   109.010000     0.000000     0.0     0.000000   \n",
       "max    8417.000000   265.320000     1.000000     0.0     1.000000   \n",
       "\n",
       "               X13          X14          X15          X16          X17  ...  \\\n",
       "count  4209.000000  4209.000000  4209.000000  4209.000000  4209.000000  ...   \n",
       "mean      0.057971     0.428130     0.000475     0.002613     0.007603  ...   \n",
       "std       0.233716     0.494867     0.021796     0.051061     0.086872  ...   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
       "75%       0.000000     1.000000     0.000000     0.000000     0.000000  ...   \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000  ...   \n",
       "\n",
       "              X375         X376         X377         X378         X379  \\\n",
       "count  4209.000000  4209.000000  4209.000000  4209.000000  4209.000000   \n",
       "mean      0.318841     0.057258     0.314802     0.020670     0.009503   \n",
       "std       0.466082     0.232363     0.464492     0.142294     0.097033   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "75%       1.000000     0.000000     1.000000     0.000000     0.000000   \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "\n",
       "              X380         X382         X383         X384         X385  \n",
       "count  4209.000000  4209.000000  4209.000000  4209.000000  4209.000000  \n",
       "mean      0.008078     0.007603     0.001663     0.000475     0.001426  \n",
       "std       0.089524     0.086872     0.040752     0.021796     0.037734  \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "75%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000  \n",
       "\n",
       "[8 rows x 370 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Generate descriptive statistics. (Count, Mean, std, min, max, 20%,50%,75%)\n",
    "## Only integer and floating columns are tatken into consideration\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "eaea8f79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>y</th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X8</th>\n",
       "      <th>...</th>\n",
       "      <th>X375</th>\n",
       "      <th>X376</th>\n",
       "      <th>X377</th>\n",
       "      <th>X378</th>\n",
       "      <th>X379</th>\n",
       "      <th>X380</th>\n",
       "      <th>X382</th>\n",
       "      <th>X383</th>\n",
       "      <th>X384</th>\n",
       "      <th>X385</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>130.81</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>a</td>\n",
       "      <td>d</td>\n",
       "      <td>u</td>\n",
       "      <td>j</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>88.53</td>\n",
       "      <td>k</td>\n",
       "      <td>t</td>\n",
       "      <td>av</td>\n",
       "      <td>e</td>\n",
       "      <td>d</td>\n",
       "      <td>y</td>\n",
       "      <td>l</td>\n",
       "      <td>o</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>281</td>\n",
       "      <td>90.54</td>\n",
       "      <td>k</td>\n",
       "      <td>c</td>\n",
       "      <td>ac</td>\n",
       "      <td>c</td>\n",
       "      <td>d</td>\n",
       "      <td>j</td>\n",
       "      <td>d</td>\n",
       "      <td>j</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>444</td>\n",
       "      <td>87.66</td>\n",
       "      <td>k</td>\n",
       "      <td>c</td>\n",
       "      <td>ac</td>\n",
       "      <td>c</td>\n",
       "      <td>d</td>\n",
       "      <td>i</td>\n",
       "      <td>j</td>\n",
       "      <td>j</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>1451</td>\n",
       "      <td>88.55</td>\n",
       "      <td>k</td>\n",
       "      <td>c</td>\n",
       "      <td>ac</td>\n",
       "      <td>a</td>\n",
       "      <td>d</td>\n",
       "      <td>af</td>\n",
       "      <td>i</td>\n",
       "      <td>j</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1322</th>\n",
       "      <td>2616</td>\n",
       "      <td>88.63</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>d</td>\n",
       "      <td>d</td>\n",
       "      <td>ac</td>\n",
       "      <td>h</td>\n",
       "      <td>s</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1352</th>\n",
       "      <td>2676</td>\n",
       "      <td>113.82</td>\n",
       "      <td>k</td>\n",
       "      <td>c</td>\n",
       "      <td>ac</td>\n",
       "      <td>c</td>\n",
       "      <td>d</td>\n",
       "      <td>ac</td>\n",
       "      <td>g</td>\n",
       "      <td>j</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1513</th>\n",
       "      <td>3023</td>\n",
       "      <td>94.60</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>d</td>\n",
       "      <td>d</td>\n",
       "      <td>ad</td>\n",
       "      <td>h</td>\n",
       "      <td>s</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1625</th>\n",
       "      <td>3260</td>\n",
       "      <td>113.87</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>d</td>\n",
       "      <td>d</td>\n",
       "      <td>ad</td>\n",
       "      <td>h</td>\n",
       "      <td>s</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1680</th>\n",
       "      <td>3368</td>\n",
       "      <td>91.76</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>d</td>\n",
       "      <td>d</td>\n",
       "      <td>ae</td>\n",
       "      <td>h</td>\n",
       "      <td>s</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1868</th>\n",
       "      <td>3738</td>\n",
       "      <td>105.64</td>\n",
       "      <td>k</td>\n",
       "      <td>v</td>\n",
       "      <td>at</td>\n",
       "      <td>d</td>\n",
       "      <td>d</td>\n",
       "      <td>ae</td>\n",
       "      <td>h</td>\n",
       "      <td>s</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11 rows × 378 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ID       y X0 X1  X2 X3 X4  X5 X6 X8  ...  X375  X376  X377  X378  \\\n",
       "0        0  130.81  k  v  at  a  d   u  j  o  ...     0     0     1     0   \n",
       "1        6   88.53  k  t  av  e  d   y  l  o  ...     1     0     0     0   \n",
       "131    281   90.54  k  c  ac  c  d   j  d  j  ...     0     0     1     0   \n",
       "220    444   87.66  k  c  ac  c  d   i  j  j  ...     0     0     1     0   \n",
       "738   1451   88.55  k  c  ac  a  d  af  i  j  ...     0     0     1     0   \n",
       "1322  2616   88.63  k  v  at  d  d  ac  h  s  ...     1     0     0     0   \n",
       "1352  2676  113.82  k  c  ac  c  d  ac  g  j  ...     0     0     1     0   \n",
       "1513  3023   94.60  k  v  at  d  d  ad  h  s  ...     1     0     0     0   \n",
       "1625  3260  113.87  k  v  at  d  d  ad  h  s  ...     1     0     0     0   \n",
       "1680  3368   91.76  k  v  at  d  d  ae  h  s  ...     1     0     0     0   \n",
       "1868  3738  105.64  k  v  at  d  d  ae  h  s  ...     1     0     0     0   \n",
       "\n",
       "      X379  X380  X382  X383  X384  X385  \n",
       "0        0     0     0     0     0     0  \n",
       "1        0     0     0     0     0     0  \n",
       "131      0     0     0     0     0     0  \n",
       "220      0     0     0     0     0     0  \n",
       "738      0     0     0     0     0     0  \n",
       "1322     0     0     0     0     0     0  \n",
       "1352     0     0     0     0     0     0  \n",
       "1513     0     0     0     0     0     0  \n",
       "1625     0     0     0     0     0     0  \n",
       "1680     0     0     0     0     0     0  \n",
       "1868     0     0     0     0     0     0  \n",
       "\n",
       "[11 rows x 378 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Returns only the result which satisfies this condition\n",
    "df[df['X0'] == 'k']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ee5f85a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = ('col1,col2,col3\\n'\n",
    "       'a,b,c\\n'\n",
    "       'd,e,f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "dea1514b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import StringIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "2b5b6ae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>b</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d</td>\n",
       "      <td>e</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  col1 col2 col3\n",
       "0    a    b    c\n",
       "1    d    e    f"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(StringIO(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c4c16f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col1</th>\n",
       "      <th>col3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  col1 col3\n",
       "0    a    c\n",
       "1    d    f"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## With the help of parameters of read_csv, we can make pandas read only specific columns\n",
    "pd.read_csv(StringIO(data),usecols = ['col1','col3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "4717996b",
   "metadata": {},
   "outputs": [],
   "source": [
    "int_data = ('col1,col2,col3\\n'\n",
    "       '1,2,3\\n'\n",
    "       '4,5,6\\n'\n",
    "       '7,8,9')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "3c6ea44e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col1</th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   col1  col2  col3\n",
       "0     1   2.0     3\n",
       "1     4   5.0     6\n",
       "2     7   8.0     9"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(StringIO(int_data),dtype={'col1':int,'col2':float})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ec8ca737",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col2</th>\n",
       "      <th>col3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      col2  col3\n",
       "col1            \n",
       "1        2     3\n",
       "4        5     6\n",
       "7        8     9"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Make your own index in the dataframe\n",
    "## i.e instead of pandas creating the index iteratively. You can use your own index\n",
    "index_data = ('col1,col2,col3\\n'\n",
    "       '1,2,3\\n'\n",
    "       '4,5,6\\n'\n",
    "       '7,8,9')\n",
    "## You can tell the column number which you want to use as the index using the index_col parameter\n",
    "## In this example, the column 0 values will be used as the index\n",
    "pd.read_csv(StringIO(index_data), index_col = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "effbe9ec",
   "metadata": {},
   "source": [
    "## Read JSON to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "508400ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data = '{\"First Name\" : \"Palaniappan\",\"Last Name\" : \"Muthuraman\",\"Place\" : \"Paderborn\",\"Studies\" : [{\"Bachelors\" : \"2015-2019\",\"Masters\" : \"2021-2024\"}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "86134cc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_df = pd.read_json(Data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "c3383026",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>First Name</th>\n",
       "      <th>Last Name</th>\n",
       "      <th>Place</th>\n",
       "      <th>Studies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Palaniappan</td>\n",
       "      <td>Muthuraman</td>\n",
       "      <td>Paderborn</td>\n",
       "      <td>{'Bachelors': '2015-2019', 'Masters': '2021-20...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    First Name   Last Name      Place  \\\n",
       "0  Palaniappan  Muthuraman  Paderborn   \n",
       "\n",
       "                                             Studies  \n",
       "0  {'Bachelors': '2015-2019', 'Masters': '2021-20...  "
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "0a45b431",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"First Name\":{\"0\":\"Palaniappan\"},\"Last Name\":{\"0\":\"Muthuraman\"},\"Place\":{\"0\":\"Paderborn\"},\"Studies\":{\"0\":{\"Bachelors\":\"2015-2019\",\"Masters\":\"2021-2024\"}}}'"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_df.to_json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "6206fa11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[{\"First Name\":\"Palaniappan\",\"Last Name\":\"Muthuraman\",\"Place\":\"Paderborn\",\"Studies\":{\"Bachelors\":\"2015-2019\",\"Masters\":\"2021-2024\"}}]'"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_df.to_json(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "ddb6ae03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"0\":{\"First Name\":\"Palaniappan\",\"Last Name\":\"Muthuraman\",\"Place\":\"Paderborn\",\"Studies\":{\"Bachelors\":\"2015-2019\",\"Masters\":\"2021-2024\"}}}'"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_df.to_json(orient='index')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "018f1401",
   "metadata": {},
   "source": [
    "## Reading HTML Content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "330f1017",
   "metadata": {},
   "outputs": [],
   "source": [
    "html_string = \"\"\"\n",
    "<table>\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th>date</th>\n",
    "      <th>name</th>\n",
    "      <th>year</th>\n",
    "      <th>cost</th>\n",
    "      <th>region</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <td>2020-01-01</td>\n",
    "      <td>Jenny</td>\n",
    "      <td>1998</td>\n",
    "      <td>0.2</td>\n",
    "      <td>South</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>2020-01-02</td>\n",
    "      <td>Alice</td>\n",
    "      <td>1992</td>\n",
    "      <td>-1.34</td>\n",
    "      <td>East</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>2020-01-03</td>\n",
    "      <td>Tomas</td>\n",
    "      <td>1982</td>\n",
    "      <td>1.00023</td>\n",
    "      <td>South</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "9cd044bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "##There is an issue with installation of lxml, hence commenting the code\n",
    "##dfs = pd.read_html(html_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9747112c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}