MAINT: update munging

examples/Carbon_Flux.ipynb

@@ -43,7 +43,8 @@
    "outputs": [],
    "source": [
     "# %read in FluxNet CSVs and replace NaNs with zeros\n",
-    "\n",
+    "import numpy as np\n",
+    "import datetime\n",
     "import os\n",
     "sites = [fname.split('_')[1] for fname in os.listdir('./flux_data/dailies/')]\n",
     "\n",
@@ -55,24 +56,62 @@
     "        return None, None\n",
     "    return float(location['lat']), float(location['lon'])\n",
     "\n",
+    "def _parse_days(integer):\n",
+    "    \"\"\" `integer` date as `20180704` to represent July 4th, 2018\"\"\"\n",
+    "    x = str(integer)\n",
+    "    d = {'year': int(x[:4]), 'month': int(x[4:6]), 'day': int(x[6:])}\n",
+    "    day_of_year = datetime.datetime(d['year'], d['month'], d['day']).timetuple().tm_yday\n",
+    "    return day_of_year\n",
+    "\n",
+    "def clean(df, timestamp_col=\"TIMESTAMP\", keep=[], drop=[], predict=''):\n",
+    "    limit = -9990\n",
+    "    for i in range(50):\n",
+    "        df = df.replace(limit - i, np.nan)\n",
+    "    \n",
+    "    assert all([col in df.columns for col in drop])\n",
+    "    df.drop(columns=drop, inplace=True)\n",
+    "    assert not all([col in df.columns for col in drop])\n",
+    "    for col in keep:\n",
+    "        if col not in df.columns:\n",
+    "            if 'SWC_F' in col or 'TS_F' in col:\n",
+    "                df[col] = 0\n",
+    "    \n",
+    "    df = df.fillna(0)\n",
+    "    df['DOY'] = df['TIMESTAMP'].apply(_parse_days)  \n",
+    "    df.pop('TIMESTAMP')\n",
+    "    X = df[keep]\n",
+    "    y = df[predict]\n",
+    "    return X, y\n",
     "\n",
     "def load_fluxnet_site(site):\n",
-    "    # dataRaw(dataRaw <= -9990) = 0/0;\n",
+    "    # dataRaw(dataRaw <= -9990) = 0/0 (is NaN?)\n",
     "    # NaN -> zero\n",
     "    prefix = 'FLX_{site}_FLUXNET'.format(site=site)\n",
-    "    filename = [fname for fname in os.listdir('./flux_data/dailies/') if fname.startswith(prefix)][0]\n",
+    "    filenames = [fname for fname in os.listdir('./flux_data/dailies/')\n",
+    "                if fname.startswith(prefix)]\n",
+    "    assert len(filenames) == 1\n",
+    "    filename = filenames[0]\n",
+    "    \n",
     "    raw_daily = pd.read_csv('./flux_data/dailies/{filename}'.format(filename=filename))    \n",
-    "    derived_cols = ['YEAR','DOY']\n",
-    "    fluxcols = ['P_ERA','TA_ERA','PA_ERA','SW_IN_ERA','LW_IN_ERA'\n",
-    "    ,'WS_ERA','LE_F_MDS','H_F_MDS','NEE_CUT_USTAR50','NEE_VUT_USTAR50',\n",
-    "    'SWC_F_MDS_1','SWC_F_MDS_2','SWC_F_MDS_3','TS_F_MDS_1','TS_F_MDS_2'\n",
-    "    'TS_F_MDS_3','VPD_ERA','GPP_DT_VUT_USTAR50','GPP_DT_CUT_USTAR50']\n",
-    "    available = [col for col in fluxcols if col in raw_daily.columns]\n",
-    "    return raw_daily[['TIMESTAMP']+available]\n",
-    "\n",
-    "\n",
-    "def map_to_days(data):\n",
-    "    return data"
+    "    \n",
+    "    keep =  ['P_ERA',\n",
+    "             'TA_ERA',\n",
+    "             'PA_ERA',\n",
+    "             'SW_IN_ERA',\n",
+    "             'LW_IN_ERA',\n",
+    "             'WS_ERA',\n",
+    "             'SWC_F_MDS_1', 'SWC_F_MDS_2', 'SWC_F_MDS_3',\n",
+    "             'TS_F_MDS_1', 'TS_F_MDS_2', 'TS_F_MDS_3',\n",
+    "             'VPD_ERA',\n",
+    "             'DOY']\n",
+    "    drop = [\"GPP_DT_VUT_USTAR50\",\n",
+    "            \"GPP_DT_CUT_USTAR50\",\n",
+    "            \"LE_F_MDS\",\n",
+    "            \"H_F_MDS\"]\n",
+    "    predict = [\"NEE_CUT_USTAR50\",\n",
+    "               \"NEE_VUT_USTAR50\"]\n",
+    "    X, y = clean(raw_daily, keep=keep, drop=drop, predict=predict[0])\n",
+    "    return X, y"
    ]
   },
   {
@@ -81,9 +120,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dailies = load_fluxnet_site(sites[0])\n",
+    "X, y = load_fluxnet_site(sites[0])\n",
     "print('Example FLUXNET data for site {site}:'.format(site=sites[0]))\n",
-    "dailies.head()"
+    "print('Observations: {}, variables per observation: {}'.format(*X.shape))\n",
+    "X.describe()"
    ]
   },
   {
@@ -158,7 +198,7 @@
    "outputs": [],
    "source": [
     "import scipy.io\n",
-    "rsif = scipy.io.loadmat('RSIF_2007_2016_05N_01L.mat')"
+    "rsif = scipy.io.loadmat('./flux_data/RSIF_2007_2016_05N_01L.mat')"
    ]
   },
   {