Merge pull request #38 from NCAR/alli_dev

Alli dev

README.rst

@@ -18,7 +18,7 @@ CESM simulations.
 
 
 :AUTHORS: Haiying Xu, Allison Baker, Daniel Milroy, Dorit Hammerling
-:COPYRIGHT: 2015-2022 University Corporation for Atmospheric Research
+:COPYRIGHT: 2015-2023 University Corporation for Atmospheric Research
 :LICENSE: Apache 2.0
 
 

der_test_pyEnsSumMPAS.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -l
+#PBS -A NTDD0004
+#PBS -N ensSumM
+#PBS -q main
+#PBS -l select=1:ncpus=36:mpiprocs=36
+#PBS -l walltime=0:20:00
+#PBS -j oe
+#PBS -M [email protected]
+
+module load conda
+conda activate npl
+
+mpiexec -n 36 -ppn 36 python pyEnsSumMPAS.py --esize 100 --indir /glade/p/cisl/asap/abaker/mpas/large_ens  --sumfile mpas_sumt4.nc  --tslice 4 --tag v7.1 --model mpas  --mach cheyenne --verbose --jsonfile empty_excluded.json

docs/source/pyEnsSum.rst

@@ -60,7 +60,7 @@ To use pyEnsSum:
        --tslice <num>       : the index into the time dimension (default = 1)
        --mach <name>        : Machine name used in the metadata (default = cheyenne)
        --jsonfile <fname>   : Jsonfile to provide that a list of variables that will be excluded
-                               or included  (default = exclude_empty.json)
+                               (default = exclude_empty.json)
        --mpi_disable        : Disable mpi mode to run in serial (off by default)
        --fIndex <num>       : Use this to start at ensemble member <num> instead of 000 (so
                               ensembles with numbers less than <num> are excluded from summary file)
@@ -98,21 +98,20 @@ Notes:
 
 7. You must specify a json file (via ``--jsonfile``) that indicates
    the variables in the ensemble
-   output files that you want to include or exclude from the summary file
-   statistics (see the example json files).  We recommend excluding variables, as
-   this is typically less work and pyEnsSum will let you know if you have not
+   output files that you want to exclude from the summary file
+   statistics (see the example json files).  The pyEnsSum routine
+   will let you know if you have not
    listed variables that need to be excluded (see next note).  Keep in mind that
    you must have *fewer* variables included than ensemble members.
 
 8. *IMPORTANT:* If there are variables that need to be excluded (that are not in
-   the .json file  already), pyEnsSum will exit early and provide a list of the
-   variables to exclude in the output.  These should be added to your exclude
-   variable list  (or removed from an include list), and then pyEnsSum can
-   be re-run.  Note that additional problematic variables may be found by
-   pyEnsSum as variables are detected in three stages. (First any variables that
-   are constant across the ensemble are identified.  Once these are removed,
-   linearly dependant variables are indentified for removal. Finally, variables
-   that are not constant but have very few unique values are identified.)
+   the .json file  already) for the summary to be generated, pyEnsSum will list these
+   variables in the output.  These variables will also be added to a copy of
+   your exclude variable list (prefixed with "NEW.") for future reference and use.
+   The summary file will be geberated with all listed variables excluded.
+   Note that the following types of variables will be removed:  any variables that
+   are constant across the ensemble, are not floating-point (e.g., integer),
+   are linearly dependant, or have very few (< 3%) unique values.
 
 
 Example:

mpas_ex_with_pv_vars.json

@@ -0,0 +1,31 @@
+{
+  "ExcludedVar": [
+    "i_rainc",
+    "i_rainnc",
+    "xice",
+    "xtime",
+    "xland",
+    "vegfra",
+    "depv_dt_diab",
+    "kpbl",
+    "iLev_DT",
+    "u",
+    "sst",
+    "u_pv",
+    "v_pv",
+    "theta_pv",
+    "vort_pv",
+    "depv_dt_lw",
+    "depv_dt_lw",
+    "depv_dt_sw",
+    "depv_dt_bl",
+    "depv_dt_cu",
+    "depv_dt_mix",
+    "dtheta_dt_mp",
+    "depv_dt_mp",
+    "depv_dt_fric",
+    "depv_dt_diab_pv",
+    "depv_dt_fric_pv",
+    "ertel_pv"
+  ]
+}

notebooks/ab_mpas_data.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "id": "92b3b345-f06f-4752-9a66-50739f398946",
    "metadata": {},
    "outputs": [],
@@ -12,7 +12,8 @@
     "import netCDF4 as nc\n",
     "import numpy as np\n",
     "\n",
-    "filedir = '/glade/scratch/abaker/mpas_hist/'"
+    "filedir = '/glade/scratch/abaker/mpas_hist/'\n",
+    "pydir = '/glade/u/home/abaker/repos/PyCECT/'"
    ]
   },
   {
@@ -146,11 +147,58 @@
     "in_files"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "8217d907-6824-46bb-b5ea-ef170dc835dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sumfile_o = nc.Dataset(pydir + 'mpas_sum_ts12.nc', 'r')\n",
+    "sumfile_n = nc.Dataset(pydir + 'mpas_sum_ts12_new.nc', 'r')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "0fd2e582-a39a-4409-b4da-cd13aa15e4d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vars_n = sumfile_n.variables\n",
+    "vars_o = sumfile_o.variables\n",
+    "gm_n = vars_n[\"global_mean\"][:, :]\n",
+    "gm_o = vars_o[\"global_mean\"][:, :]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "478930f9-c1b6-4ab4-82d1-e5eaa849d955",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.0"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.max(gm_n - gm_o)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 4,
    "id": "f4ea1560-03d7-4f4a-a34d-12fea6b2b3a4",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "myfile = nc.Dataset(filedir + in_files[0], 'r')\n",
@@ -521,9 +569,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "NPL (conda)",
+   "display_name": "NPL 2023b",
    "language": "python",
-   "name": "npl-conda"
+   "name": "npl-2023b"
   },
   "language_info": {
    "codemirror_mode": {
@@ -535,7 +583,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.12"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,