10_Exporting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10.1 Figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fires = pd.read_csv(\"data/VIIRSNDE_global2018312.v1.0.txt\")\n",
    "bins10MW = np.arange(0, 500, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[5,5]) \n",
    "plt.hist(fires['frp(MW)'],  bins=bins10MW)\n",
    "\n",
    "plt.savefig('histogram.png',  bbox_inches='tight') \n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10.2 Text Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = ['GOES-16', 'IceSat-2', 'Himawari']\n",
    "agency = ['NOAA', 'NASA', 'JAXA']\n",
    "orbit = ['GEO', 'LEO', 'GEO']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "with open('satellite.txt','w', newline='') as csvfile:\n",
    "    writer = csv.writer(csvfile, delimiter=',')\n",
    "    \n",
    "    # adds a header\n",
    "    writer.writerow(['name', 'agency', 'orbit'])\n",
    "    \n",
    "    # Writes each element as a single row\n",
    "    for i in range(0, len(name)):\n",
    "        writer.writerow([name[i], agency[i], orbit[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "satellites = pd.DataFrame({'name': name,\n",
    "                   'agency': agency,\n",
    "                   'orbit': orbit})\n",
    "\n",
    "satellites.to_csv('satellites.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10.3 Pickling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "pkl.dump(satellites, open(\"satellites.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>agency</th>\n",
       "      <th>orbit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GOES-16</td>\n",
       "      <td>NOAA</td>\n",
       "      <td>GEO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IceSat-2</td>\n",
       "      <td>NASA</td>\n",
       "      <td>LEO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Himawari</td>\n",
       "      <td>JAXA</td>\n",
       "      <td>GEO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       name agency orbit\n",
       "0   GOES-16   NOAA   GEO\n",
       "1  IceSat-2   NASA   LEO\n",
       "2  Himawari   JAXA   GEO"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "satellites2 = pkl.load(open(\"satellites.p\", \"rb\"))\n",
    "satellites2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10.4 NumPy binary files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez('satnames', name=name, agency=agency, orbit=orbit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['name', 'agency', 'orbit']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "npzfile = np.load('satnames.npz')\n",
    "npzfile.files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['GOES-16', 'IceSat-2', 'Himawari'], dtype='<U8')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "npzfile['name']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10.5 NetCDF\n",
    "### 10.5.1 Using the netCDF4 package to create netCDF4 files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from netCDF4 import Dataset\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get file names, exit if no files are found\n",
    "filenames = glob('data/aod/JRR-AOD_v2r0_j01_s2019*.nc')\n",
    "\n",
    "# Import datasets\n",
    "file_id = Dataset(filenames[0], mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Latitude ('Rows', 'Columns')\n",
      "Longitude ('Rows', 'Columns')\n",
      "AOD550 ('Rows', 'Columns')\n",
      "QCAll ('Rows', 'Columns')\n"
     ]
    }
   ],
   "source": [
    "for var in [\"Latitude\", \"Longitude\", \"AOD550\", \"QCAll\"]:\n",
    "    print(var, file_id.variables[var].dimensions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns 3200\n",
      "Rows 768\n"
     ]
    }
   ],
   "source": [
    "for dim in ['Columns', 'Rows']:\n",
    "    print(dim, len(file_id.dimensions[dim]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "768 3200 3\n"
     ]
    }
   ],
   "source": [
    "cols = len(file_id.dimensions['Columns'])\n",
    "rows = len(file_id.dimensions['Rows'])\n",
    "num_files = len(filenames)\n",
    "file_id.close()\n",
    "\n",
    "print(rows, cols, num_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create output file name\n",
    "output_filename = \"JRR-AOD-combined.nc\"\n",
    "\n",
    "# Open a new netCDF file \n",
    "rootgrp = Dataset(output_filename, \"w\", format=\"NETCDF4\")\n",
    "rootgrp.description = \"Combined NOAA-20 AOD swaths for Alberta Fire\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format the latitude and longitude variables\n",
    "lat = rootgrp.createDimension(\"lat\", rows*num_files)\n",
    "lon = rootgrp.createDimension(\"lon\", cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "latitudes = rootgrp.createVariable(\"Latitude\",\"f4\", (\"lat\",\"lon\"), \\\n",
    "           zlib = True, least_significant_digit = 2)\n",
    "longitudes = rootgrp.createVariable(\"Longitude\",\"f4\", (\"lat\",\"lon\"), \\\n",
    "           zlib = True, least_significant_digit = 2)\n",
    "variable = rootgrp.createVariable('AOD550', \"f4\", (\"lat\",\"lon\"), \\\n",
    "           zlib = True, least_significant_digit = 2, fill_value = -999.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "latitudes.units = \"degrees north\"\n",
    "longitudes.units = \"degrees east\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = np.zeros((rows*num_files, cols))\n",
    "lats = np.zeros((rows*num_files, cols))\n",
    "lons = np.zeros((rows*num_files, cols))\n",
    "\n",
    "for filenum, filename in enumerate(filenames):\n",
    "    file_id = Dataset(filename, mode='r')\n",
    "    \n",
    "    i1 = rows*filenum\n",
    "    i2 = rows*(filenum+1)\n",
    "    \n",
    "    # Get coordinates\n",
    "    lats[i1:i2, 0:cols] = file_id.variables['Latitude'][:,:]\n",
    "    lons[i1:i2, 0:cols] = file_id.variables['Longitude'][:,:]\n",
    "        \n",
    "    # Create and fill variables\n",
    "    value = file_id.variables['AOD550'][:,:]\n",
    "\n",
    "    # Simple binary quality flag    \n",
    "    dqf = file_id.variables[ 'QCAll'][:,:]\n",
    "    value[ dqf != 0 ] = -999.9\n",
    "    \n",
    "    var[i1:i2, :] = value[:,:]\n",
    "\n",
    "    file_id.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Populate lat/lon/var\n",
    "latitudes[:,:] = lats\n",
    "longitudes[:,:] = lons\n",
    "variable[:,:] = var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Close original/modified netCDF file\n",
    "rootgrp.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10.5.2 Using Xarray to create netCDF files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xarray as xr\n",
    "\n",
    "fname = 'data/aod/JRR-AOD*.nc'\n",
    "aod = xr.open_mfdataset(fname, concat_dim='Rows', combine='nested')\n",
    "filtered = aod['AOD550'].where(aod['QCAll'] == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered.to_netcdf('JRR-AOD-xarray-combined.nc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(filtered.attrs['units'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "earthsciviz",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "vscode": {
   "interpreter": {
    "hash": "48fc10cb14791ea88ba2696b1818b1bf7d4d99ed63d4104785d39779bf72a716"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}