ReadWrite.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the current version.\n",
    "\n",
    "Running with an ERDDAP URL was many time slower to load.\n",
    "\n",
    "This version does each plot individually and composes them in a GridSpec panel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import utility as u\n",
    "import numpy as np\n",
    "import datetime\n",
    "import colorcet as cc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('latest.csv', skiprows=[1])\n",
    "df.sort_values(['time','observation_depth'], ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove any row that has no surface observation for the 7 surface variables we are plotting.\n",
    "surface_cols = ('latitude', \n",
    "                 'longitude', \n",
    "                 'time', \n",
    "                 'observation_depth', \n",
    "                 'platform_type', \n",
    "                 'platform_code', \n",
    "                 'sst', \n",
    "                 'slp',\n",
    "                 'atmp',\n",
    "                 'winddir',\n",
    "                 'windspd',\n",
    "                 'dewpoint',\n",
    "                 'clouds')\n",
    "\n",
    "# Some platforms report ztmp with observation_depth = 0.0 which is really sst.\n",
    "# So to get all surface obs you have to segment by platform type\n",
    "surface_platforms = ['TROPICAL MOORED BUOYS',\n",
    "                     'C-MAN WEATHER STATIONS',\n",
    "                     'DRIFTING BUOYS (GENERIC)',\n",
    "                     'ICE BUOYS',\n",
    "                     'MOORED BUOYS (GENERIC)',\n",
    "                     'RESEARCH',\n",
    "                     'SHIPS (GENERIC)',\n",
    "                     'SHORE AND BOTTOM STATIONS (GENERIC)',\n",
    "                     'TIDE GAUGE STATIONS (GENERIC)',\n",
    "                     'TSUNAMI WARNING STATIONS',\n",
    "                     'UNKNOWN',\n",
    "                     'UNMANNED SURFACE VEHICLE',\n",
    "                     'VOLUNTEER OBSERVING SHIPS',\n",
    "                     'VOLUNTEER OBSERVING SHIPS (GENERIC)',\n",
    "                     'VOSCLIM',\n",
    "                     'WEATHER AND OCEAN OBS',\n",
    "                     'WEATHER BUOYS',\n",
    "                     'WEATHER OBS']\n",
    "\n",
    "# Removes rows with other platform types\n",
    "surface = df[df.platform_type.isin(surface_platforms)]\n",
    "# Removes columns that are not metdadata or a desired surface variable\n",
    "surface = surface.loc[:, surface_cols]\n",
    "# Removes any row that does not have at least 1 surface observation that is not NaN\n",
    "surface.dropna(subset=['sst', 'slp', 'atmp', 'winddir', 'windspd', 'dewpoint', 'clouds'], how='all', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "depth_platforms = ['TROPICAL MOORED BUOYS',\n",
    "                   'AUTONOMOUS PINNIPEDS',\n",
    "                   'CLIMATE REFERENCE MOORED BUOYS',\n",
    "                   'GLIDERS',\n",
    "                   'ICE BUOYS',\n",
    "                   'OCEAN TRANSPORT STATIONS (GENERIC)',\n",
    "                   'PROFILING FLOATS AND GLIDERS (GENERIC)',\n",
    "                   'SHORE AND BOTTOM STATIONS (GENERIC)']\n",
    "\n",
    "# Keep only rows with platforms that report data at depth\n",
    "depth = df[df.platform_type.isin(depth_platforms)]\n",
    "# Keep only metadata and depth observation columns (zsal and ztmp)\n",
    "depth_cols = ('latitude', 'longitude', 'time', 'observation_depth', 'platform_type', 'platform_code', 'zsal', 'ztmp')\n",
    "depth = depth.loc[:, depth_cols]\n",
    "# Drop any row that has both ztmp and zsal set to NaN\n",
    "depth.dropna(subset=['ztmp', 'zsal'], how='all', inplace=True)\n",
    "# Convert time string to time value, keeping a copy\n",
    "depth.loc[:, 'time_val'] = depth['time']\n",
    "depth.loc[:, 'time'] = pd.to_datetime(depth['time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "surface_max = surface.drop_duplicates(['platform_code'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "surface_locs = surface_max.loc[:,('latitude', 'longitude', 'platform_type', 'platform_code')]\n",
    "surface_locs.sort_values(['platform_type'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make time the index for the surface data since all plots are timeseries.\n",
    "surface['time_val'] = surface['time']\n",
    "surface.loc[:, 'time'] = pd.to_datetime(surface['time'])\n",
    "surface.set_index('time', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_to_find_surface = depth.sort_values(['time','observation_depth'])\n",
    "depth_ob_locs = sort_to_find_surface.drop_duplicates(['time'])\n",
    "depth_max = depth_ob_locs.drop_duplicates(['platform_code'])\n",
    "depth_locs = depth_max.loc[:,('latitude', 'longitude', 'platform_type', 'platform_code')]\n",
    "depth_locs.sort_values(['platform_type'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "depth_locs.to_pickle('depth_locations_latest.pkl')\n",
    "surface_locs.to_pickle('surface_locations_latest.pkl')\n",
    "surface.to_pickle('surface_data_latest.pkl')\n",
    "depth.to_pickle('depth_data_latest.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}