data_cdf_df.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T11:49:36.824035Z",
     "start_time": "2020-02-17T11:49:35.723027Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "import cdflib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T11:49:36.831225Z",
     "start_time": "2020-02-17T11:49:36.827433Z"
    }
   },
   "outputs": [],
   "source": [
    "base_f_loc = '/storage/silver/stfc_cg/hf832176/data/THEMIS/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic commands:\n",
    "\n",
    "data = some .cdf file.\n",
    "\n",
    "data.cdf_info() - lists variables and more\n",
    "\n",
    "data.varattsget() - lists variables\n",
    "\n",
    "data.varattsget(variable_name) - lists variable info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:57:45.179908Z",
     "start_time": "2020-02-17T12:57:45.177160Z"
    }
   },
   "outputs": [],
   "source": [
    "spacecraft = 'the'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:58:04.454279Z",
     "start_time": "2020-02-17T12:58:04.450813Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_file_paths(base_f_loc, instrument, spacecraft):\n",
    "    f_path = base_f_loc+instrument+'/'+spacecraft\n",
    "    files = !ls {f_path}?*.cdf\n",
    "    return files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:58:24.940080Z",
     "start_time": "2020-02-17T12:58:24.938075Z"
    }
   },
   "outputs": [],
   "source": [
    "# files[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:58:45.162628Z",
     "start_time": "2020-02-17T12:58:45.160596Z"
    }
   },
   "outputs": [],
   "source": [
    "# instrument = 'sst'\n",
    "# files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "# data = cdflib.CDF('/storage/silver/stfc_cg/hf832176/data/THEMIS/sst/tha_l2_sst_20070926_v01.cdf')#files[100])\n",
    "# # eflux = data.varget(spacecraft+'_psef_en_eflux')\n",
    "# # data.varattsget()\n",
    "# data.cdf_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:59:04.796982Z",
     "start_time": "2020-02-17T12:59:04.794966Z"
    }
   },
   "outputs": [],
   "source": [
    "# data.varget('tha_psef_data_quality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:59:25.959268Z",
     "start_time": "2020-02-17T12:59:25.957118Z"
    }
   },
   "outputs": [],
   "source": [
    "# data.varattsget('thb_peef_en_eflux')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T12:59:45.795442Z",
     "start_time": "2020-02-17T12:59:45.793081Z"
    }
   },
   "outputs": [],
   "source": [
    "# data.varattsget('tha_psef_en_eflux')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center>SST data: Spectra</center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:04:29.009037Z",
     "start_time": "2020-02-17T13:00:07.871798Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File failed to load: /storage/silver/stfc_cg/hf832176/data/THEMIS/sst/the_l2_sst_20070927_v01.cdf\n",
      "File failed to load: /storage/silver/stfc_cg/hf832176/data/THEMIS/sst/the_l2_sst_20070928_v01.cdf\n",
      "File failed to load: /storage/silver/stfc_cg/hf832176/data/THEMIS/sst/the_l2_sst_20070929_v01.cdf\n",
      "File failed to load: /storage/silver/stfc_cg/hf832176/data/THEMIS/sst/the_l2_sst_20071005_v01.cdf\n"
     ]
    }
   ],
   "source": [
    "instrument = 'sst'\n",
    "files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "\n",
    "sub_dfs = []\n",
    "for i in files:\n",
    "#     print(i)\n",
    "    # Load data\n",
    "    try:\n",
    "        data = cdflib.CDF(i)\n",
    "    except OSError:\n",
    "        print('File failed to load: '+i)\n",
    "        pass\n",
    "    # Read electron spectrogram\n",
    "    try:\n",
    "        eflux = data.varget(spacecraft+'_psef_en_eflux')\n",
    "        # Read spectrogram quality flag (>0 is bad)\n",
    "        quality = data.varget(spacecraft+'_psef_data_quality')\n",
    "    except ValueError:\n",
    "        print('Variable not present')\n",
    "        continue\n",
    "    \n",
    "    # Read energy levels for data\n",
    "    ene = data.varget(spacecraft+'_psef_en_eflux_yaxis')\n",
    "    e_col_names = ['E_'+str(i+1) for i in range(len(ene[0]))]\n",
    "    # Create datetimes for timestamps\n",
    "    try:\n",
    "        dti = [dt.datetime(1970,1,1) + dt.timedelta(seconds=i)\n",
    "           for i in data.varget(spacecraft+'_psef_time')]\n",
    "    except TypeError:\n",
    "        print('No data: '+i)\n",
    "    \n",
    "    if len(quality) < len(eflux):\n",
    "        quality2 = [np.nan]*len(eflux)\n",
    "        for item, idx in enumerate(quality):\n",
    "            quality2[idx] = item\n",
    "        quality = quality2\n",
    "    df = pd.DataFrame(eflux,index=dti,columns=e_col_names)\n",
    "    df['quality'] = quality\n",
    "    \n",
    "    sub_dfs.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:06:36.971543Z",
     "start_time": "2020-02-17T13:06:36.969179Z"
    }
   },
   "outputs": [],
   "source": [
    "# instrument = 'sst'\n",
    "# files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "# data = cdflib.CDF(files[0])\n",
    "# data.varattsget(spacecraft+'_psef_en_eflux')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:10:24.138648Z",
     "start_time": "2020-02-17T13:08:41.129547Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 4974832 entries, 2007-09-15 00:03:23.082396 to 2010-04-01 00:05:25.610651\n",
      "Data columns (total 17 columns):\n",
      " #   Column   Dtype  \n",
      "---  ------   -----  \n",
      " 0   E_1      float64\n",
      " 1   E_2      float64\n",
      " 2   E_3      float64\n",
      " 3   E_4      float64\n",
      " 4   E_5      float64\n",
      " 5   E_6      float64\n",
      " 6   E_7      float64\n",
      " 7   E_8      float64\n",
      " 8   E_9      float64\n",
      " 9   E_10     float64\n",
      " 10  E_11     float64\n",
      " 11  E_12     float64\n",
      " 12  E_13     float64\n",
      " 13  E_14     float64\n",
      " 14  E_15     float64\n",
      " 15  E_16     float64\n",
      " 16  quality  int32  \n",
      "dtypes: float64(16), int32(1)\n",
      "memory usage: 664.2 MB\n"
     ]
    }
   ],
   "source": [
    "comb_df = pd.concat(sub_dfs)\n",
    "comb_df.info()\n",
    "comb_df.to_csv(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.csv')\n",
    "comb_df.to_pickle(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:12:11.313695Z",
     "start_time": "2020-02-17T13:12:11.311440Z"
    }
   },
   "outputs": [],
   "source": [
    "# instrument = 'sst'\n",
    "# spacecraft = 'tha'\n",
    "# files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "# data = cdflib.CDF(files[0])\n",
    "# ene = data.varget(spacecraft+'_psef_en_eflux_yaxis')\n",
    "# ene[0]\n",
    "# df = pd.DataFrame([i for i in range(len(ene[0]))],index=ene[0],columns=['eV'])\n",
    "# df.to_csv(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'_energy_levels.csv')\n",
    "# df.to_pickle(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'_energy_levels.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center> STATE: Position, orientation etc. </center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:16:05.981082Z",
     "start_time": "2020-02-17T13:14:00.295697Z"
    }
   },
   "outputs": [],
   "source": [
    "instrument = 'state'\n",
    "files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "\n",
    "sub_dfs = []\n",
    "for i in files:\n",
    "    # Load data\n",
    "    try:\n",
    "        data = cdflib.CDF(i)\n",
    "    except OSError:\n",
    "        print('File failed to load: '+i)\n",
    "        pass\n",
    "    # Read GSM position\n",
    "    pos_gsm = data.varget(spacecraft+'_pos_gsm')\n",
    "    pos_gei = data.varget(spacecraft+'_pos')\n",
    "    \n",
    "    spin_ra = data.varget(spacecraft+'_spinras') # right ascension\n",
    "    spin_dec = data.varget(spacecraft+'_spindec') # declination\n",
    "    spin_per = data.varget(spacecraft+'_spinper') # spin period\n",
    "    spin_phase = data.varget(spacecraft+'_spinphase') #spin phase\n",
    "    \n",
    "    # Create datetimes for timestamps\n",
    "    try:\n",
    "        dti = [dt.datetime(1970,1,1) + dt.timedelta(seconds=i)\n",
    "           for i in data.varget(spacecraft+'_state_time')]\n",
    "    except TypeError:\n",
    "        print('No data: '+i)\n",
    "    \n",
    "    df = pd.DataFrame(pos_gsm,index=dti,columns=['pos_x_gsm','pos_y_gsm','pos_z_gsm'])\n",
    "    df['spin_ra'] = spin_ra\n",
    "    df['spin_dec'] = spin_dec\n",
    "    df['spin_per'] = spin_per\n",
    "    df['spin_phase'] = spin_phase\n",
    "    df['pos_x_gei'] = pos_gei[:,0]\n",
    "    df['pos_y_gei'] = pos_gei[:,1]\n",
    "    df['pos_z_gei'] = pos_gei[:,2]\n",
    "    \n",
    "    sub_dfs.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:17:26.088529Z",
     "start_time": "2020-02-17T13:17:26.086593Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# instrument = 'state'\n",
    "# spacecraft = 'tha'\n",
    "# files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "\n",
    "# data = cdflib.CDF(files[0])\n",
    "\n",
    "# data.varattsget('tha_pos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:19:02.113202Z",
     "start_time": "2020-02-17T13:18:47.643399Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 1052640 entries, 2007-09-15 00:00:00 to 2010-03-31 23:59:00\n",
      "Data columns (total 10 columns):\n",
      " #   Column      Non-Null Count    Dtype  \n",
      "---  ------      --------------    -----  \n",
      " 0   pos_x_gsm   1052640 non-null  float32\n",
      " 1   pos_y_gsm   1052640 non-null  float32\n",
      " 2   pos_z_gsm   1052640 non-null  float32\n",
      " 3   spin_ra     1052640 non-null  float32\n",
      " 4   spin_dec    1052640 non-null  float32\n",
      " 5   spin_per    1052640 non-null  float32\n",
      " 6   spin_phase  1052640 non-null  float32\n",
      " 7   pos_x_gei   1052640 non-null  float32\n",
      " 8   pos_y_gei   1052640 non-null  float32\n",
      " 9   pos_z_gei   1052640 non-null  float32\n",
      "dtypes: float32(10)\n",
      "memory usage: 48.2 MB\n"
     ]
    }
   ],
   "source": [
    "comb_df = pd.concat(sub_dfs)\n",
    "comb_df.info()\n",
    "comb_df.to_csv(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.csv')\n",
    "comb_df.to_pickle(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center> FGM: B-field etc. </center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:32:00.855065Z",
     "start_time": "2020-02-17T13:20:15.401807Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20070927_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20070928_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20070929_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20071005_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20081003_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20081208_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20090711_v01.cdf\n",
      "No data: /storage/silver/stfc_cg/hf832176/data/THEMIS/fgm/the_l2_fgm_20090715_v01.cdf\n"
     ]
    }
   ],
   "source": [
    "instrument = 'fgm'\n",
    "data_rate = 'l' # or 'h'\n",
    "files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "\n",
    "sub_dfs = []\n",
    "for i in files:\n",
    "    # Load data\n",
    "    try:\n",
    "        data = cdflib.CDF(i)\n",
    "    except OSError:\n",
    "        print('File failed to load: '+i)\n",
    "        pass\n",
    "    # Read GSM position\n",
    "    try:\n",
    "        b_gsm = data.varget(spacecraft+'_fg'+data_rate+'_gsm')\n",
    "    except ValueError as err:\n",
    "        print(\"Value error: {0}\".format(err))\n",
    "        print(i)\n",
    "        pass\n",
    "    except EOFError as err:\n",
    "        print(\"EOF error: {0}\".format(err))\n",
    "        print(i)\n",
    "        pass\n",
    "    \n",
    "    try:    \n",
    "        quality = data.varget(spacecraft+'_fg'+data_rate+'_gsmQ')\n",
    "    except ValueError as err:\n",
    "        print(\"Value error: {0}\".format(err))\n",
    "        print(i)\n",
    "        pass\n",
    "    except EOFError as err:\n",
    "        print(\"EOF error: {0}\".format(err))\n",
    "        print(i)\n",
    "        pass\n",
    "    \n",
    "    # Create datetimes for timestamps\n",
    "    try:\n",
    "        dti = [dt.datetime(1970,1,1) + dt.timedelta(seconds=i)\n",
    "               for i in data.varget(spacecraft+'_fg'+data_rate+'_time')]\n",
    "    except TypeError:\n",
    "        print('No data: '+i)\n",
    "    except ValueError as err:\n",
    "        print(\"Value error: {0}\".format(err))\n",
    "        print(i)\n",
    "        pass\n",
    "    except EOFError as err:\n",
    "        print(\"EOF error: {0}\".format(err))\n",
    "        print(i)\n",
    "        pass\n",
    "\n",
    "    \n",
    "    df = pd.DataFrame(b_gsm,index=dti,columns=['Bx_gsm','By_gsm','Bz_gsm'])\n",
    "    df['quality'] = quality\n",
    "    \n",
    "    sub_dfs.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:46:13.149817Z",
     "start_time": "2020-02-17T13:32:49.511543Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 138960896 entries, 2007-09-15 00:00:25.464584 to 2010-03-31 13:00:08.858704\n",
      "Data columns (total 4 columns):\n",
      " #   Column   Dtype  \n",
      "---  ------   -----  \n",
      " 0   Bx_gsm   float32\n",
      " 1   By_gsm   float32\n",
      " 2   Bz_gsm   float32\n",
      " 3   quality  object \n",
      "dtypes: float32(3), object(1)\n",
      "memory usage: 3.6+ GB\n"
     ]
    }
   ],
   "source": [
    "comb_df = pd.concat(sub_dfs)\n",
    "comb_df.info()\n",
    "comb_df.to_csv(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.csv')\n",
    "comb_df.to_pickle(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center> ESA Data: Spectra </center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T13:54:37.804767Z",
     "start_time": "2020-02-17T13:46:59.799939Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No energy data: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070927_v01.cdf\n",
      "No time data: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070927_v01.cdf\n",
      "File failed to load: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070928_v01.cdf\n",
      "No energy data: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070928_v01.cdf\n",
      "No time data: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070928_v01.cdf\n",
      "No energy data: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070929_v01.cdf\n",
      "No time data: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20070929_v01.cdf\n",
      "File failed to load: /storage/silver/stfc_cg/hf832176/data/THEMIS/esa/the_l2_esa_20071005_v01.cdf\n"
     ]
    }
   ],
   "source": [
    "instrument = 'esa'\n",
    "files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "\n",
    "sub_dfs = []\n",
    "for i in files[:]:\n",
    "    # Load data\n",
    "    try:\n",
    "        data = cdflib.CDF(i)\n",
    "    except OSError:\n",
    "        print('File failed to load: '+i)\n",
    "        pass\n",
    "    # Read electron spectrogram\n",
    "    try:\n",
    "        eflux = data.varget(spacecraft+'_peef_en_eflux')\n",
    "        # Read spectrogram quality flag (>0 is bad)\n",
    "        quality = data.varget(spacecraft+'_peef_data_quality')\n",
    "        # Read energy levels for data\n",
    "        ene = data.varget(spacecraft+'_peef_en_eflux_yaxis')\n",
    "    except ValueError:\n",
    "        print('Variable not found')\n",
    "        continue\n",
    "    \n",
    "    try:\n",
    "        e_col_names = ['E_'+str(i+1) for i in range(len(ene[0]))]\n",
    "    except TypeError:\n",
    "        print('No energy data: '+i)\n",
    "    # Create datetimes for timestamps\n",
    "    try:\n",
    "        dti = [dt.datetime(1970,1,1) + dt.timedelta(seconds=i)\n",
    "           for i in data.varget(spacecraft+'_peef_time')]\n",
    "    except TypeError:\n",
    "        print('No time data: '+i)\n",
    "    \n",
    "    df = pd.DataFrame(eflux,index=dti,columns=e_col_names)\n",
    "    df['quality'] = quality\n",
    "    \n",
    "    sub_dfs.append(df)\n",
    "    \n",
    "#     display(df.head(2))\n",
    "#     display(eflux)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T14:00:14.153341Z",
     "start_time": "2020-02-17T13:57:37.464466Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 4996745 entries, 2007-09-15 00:00:34.165417 to 2010-03-31 23:54:33.952057\n",
      "Data columns (total 33 columns):\n",
      " #   Column   Dtype  \n",
      "---  ------   -----  \n",
      " 0   E_1      float64\n",
      " 1   E_2      float64\n",
      " 2   E_3      float64\n",
      " 3   E_4      float64\n",
      " 4   E_5      float64\n",
      " 5   E_6      float64\n",
      " 6   E_7      float64\n",
      " 7   E_8      float64\n",
      " 8   E_9      float64\n",
      " 9   E_10     float64\n",
      " 10  E_11     float64\n",
      " 11  E_12     float64\n",
      " 12  E_13     float64\n",
      " 13  E_14     float64\n",
      " 14  E_15     float64\n",
      " 15  E_16     float64\n",
      " 16  E_17     float64\n",
      " 17  E_18     float64\n",
      " 18  E_19     float64\n",
      " 19  E_20     float64\n",
      " 20  E_21     float64\n",
      " 21  E_22     float64\n",
      " 22  E_23     float64\n",
      " 23  E_24     float64\n",
      " 24  E_25     float64\n",
      " 25  E_26     float64\n",
      " 26  E_27     float64\n",
      " 27  E_28     float64\n",
      " 28  E_29     float64\n",
      " 29  E_30     float64\n",
      " 30  E_31     float64\n",
      " 31  E_32     float64\n",
      " 32  quality  object \n",
      "dtypes: float64(32), object(1)\n",
      "memory usage: 1.3+ GB\n"
     ]
    }
   ],
   "source": [
    "comb_df = pd.concat(sub_dfs)\n",
    "comb_df.info()\n",
    "comb_df.to_csv(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.csv')\n",
    "comb_df.to_pickle(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T14:03:07.412331Z",
     "start_time": "2020-02-17T14:03:07.410177Z"
    }
   },
   "outputs": [],
   "source": [
    "# instrument = 'esa'\n",
    "# spacecraft = 'tha'\n",
    "# files = get_file_paths(base_f_loc, instrument, spacecraft)\n",
    "# data = cdflib.CDF(files[20])\n",
    "# ene = data.varget(spacecraft+'_peef_en_eflux_yaxis')\n",
    "# ene[0]\n",
    "# df = pd.DataFrame([i for i in range(len(ene[0]))],index=ene[0],columns=['eV'])\n",
    "# df.to_csv(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'_energy_levels.csv')\n",
    "# df.to_pickle(base_f_loc+spacecraft+'/'+spacecraft+'_'+instrument+'_energy_levels.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}