Grackle Tests

.circleci/config.yml

@@ -13,7 +13,6 @@ commands:
             echo 'export HDF5_LIB=/usr/lib/x86_64-linux-gnu' >> $BASH_ENV
             echo 'export CHARM_ROOT=$HOME/local/charm-v$CHARM_VER' >> $BASH_ENV
             echo 'export Grackle_ROOT=$HOME/local' >> $BASH_ENV
-            echo 'export GRACKLE_INPUT_DATA_DIR=$HOME/grackle/grackle_data_files/input/' >> $BASH_ENV
             # tag the tip so we can go back to it
             git tag tip
 
@@ -125,9 +124,24 @@ commands:
             # convert boolean parameter to an env var storing 0 or 1
             SKIP_TEST=$(( 0 <<# parameters.skiptest >> + 1 <</ parameters.skiptest >> ))
             USE_DOUBLE=$(( 0 <<# parameters.usedouble >> + 1 <</ parameters.usedouble >> ))
+            USE_GRACKLE=$(( 0 <<# parameters.usegrackle >> + 1 <</ parameters.usegrackle >> ))
+
+            # this is used for tests involving Grackle
+            if [[ $USE_GRACKLE == 1 ]]; then
+              GRACKLE_INPUT_DATA_DIR="$HOME/grackle/grackle_data_files/input/"
+            else
+              GRACKLE_INPUT_DATA_DIR=""
+            fi
 
             if [ ! -f << parameters.skipfile >> ]; then
-              cmake -DEnzo-E_CONFIG=linux_gcc -GNinja -DUSE_DOUBLE_PREC=<< parameters.usedouble >> -DUSE_GRACKLE=<< parameters.usegrackle >> -Bbuild -DPARALLEL_LAUNCHER_NPROC_ARG="++local;+p" -DPython3_FIND_VIRTUALENV=ONLY
+              cmake -DEnzo-E_CONFIG=linux_gcc \
+                    -GNinja \
+                    -DUSE_DOUBLE_PREC=<< parameters.usedouble >> \
+                    -DUSE_GRACKLE=<< parameters.usegrackle >> \
+                    -DGRACKLE_INPUT_DATA_DIR="$GRACKLE_INPUT_DATA_DIR" \
+                    -Bbuild \
+                    -DPARALLEL_LAUNCHER_NPROC_ARG="++local;+p" \
+                    -DPython3_FIND_VIRTUALENV=ONLY
               cmake --build build -j 4
               source $HOME/venv/bin/activate
               if [[ $SKIP_TEST != 1 ]]; then

doc/source/tests/grackle.rst

@@ -0,0 +1,37 @@
+----------------
+Grackle Tests
+----------------
+
+Tests for the method that invokes Grackle. These tests set up a
+cooling test without hydrodynamics to run many one-zone models in
+Grackle, fully sampling the density, temperature, and metallicity
+parameter space over which the chemistry and cooling/heating tables
+are valid.
+
+method_grackle_general
+======================
+
+This test compares the summary statistics computed for several grackle
+fields after a certain period of time to previously archived values.
+
+The simulation timesteps are much larger that the cooling/heating.
+This makes it more likely that separate processing elements will
+execute grackle routines at the same time (thus increasing the chances
+of exposing hypothetical problems related to Grackle & SMP mode).
+
+This test is somewhat fragile given that upgrading Grackle versions could
+conceivably alter the field values. In the future it would be better to
+replace this with a test that:
+
+1. checks out and builds a previous commit of Enzo-E
+2. runs the simulation and saves the exact field values after running the simulations
+3. checks out and builds a newer commit of Enzo-E (while leaving the build of Grackle unchanged)
+4. runs the simulation and confirms that the Grackle related field values are identical to the field values from the earlier simulation.
+
+method_grackle_cooling_dt
+=========================
+
+This test runs Grackle for a fixed number of cycles, and compares the
+final simulation time to a reference value. Each simulation timestep
+is set fraction of the minimum magnitude of the cooling/heating
+timestep.

doc/source/tests/index.rst

@@ -14,6 +14,7 @@ This testing section describes tests on enzo-e program
    collapse
    cosmology
    gravity
+   grackle
    heat
    helloworld
    hierarchy

input/Cosmology/method_cosmology.incl

@@ -324,7 +324,7 @@
                         "is_local", "default"];
          position = [ "x", "y", "z" ];
 	 velocity = [ "vx", "vy", "vz" ];
-         constants = [ "mass", "double", 2.5581226e-5 ];
+         constants = [ "mass", "default", 2.5581226e-5 ];
          group_list = [ "is_gravitating" ];
      }
  }

input/Grackle/grackle.incl

@@ -0,0 +1,212 @@
+#
+#  Grackle Test Problem:
+#
+#  Sets up a cooling test without hydrodynamics to run many
+#  one-zone models in Grackle, fully sampling the density,
+#  temperature, and metallicity parameter space over which 
+#  the chemistry and cooling/heating tables are valid.
+#
+#  When including this file to launch a simulation, be sure to:
+#    - overwrite the entry in Method:grackle:data_file parameter
+#    - specify a stopping condition
+#    - specify an output schedule for the "data" output fileset
+#      (or drop that fileset from the output list)
+
+ Boundary {
+     type = "reflecting";
+ }
+
+ Domain {
+     lower = [ 0.0, 0.0, 0.0];
+     upper = [ 1.0, 1.0, 1.0];
+ }
+
+ Units {
+    density = 1.6726219E-24;  # m_H in grams... so n = 1.0 cm^-3 for pure hydrogen
+    time    = 3.15576E13;     # 1 Myr in seconds
+    length  = 3.086E16;       # 0.01 pc in cm - does not actually matter
+ }
+
+ Field {
+     alignment = 8;
+     gamma = 1.400;
+     ghost_depth = 3;
+     list = [ "density", 
+              "internal_energy",
+              "total_energy",
+              "velocity_x",
+              "velocity_y",
+              "velocity_z",
+# purposefully commenting these out to test automatic
+# creation of needed fields
+#              "HI_density",
+#              "HII_density",
+#              "HM_density",
+#              "HeI_density",
+#              "HeII_density",
+#              "HeIII_density",
+#              "H2I_density",
+#              "H2II_density",
+#              "DI_density",
+#              "DII_density",
+#              "HDI_density",
+#              "e_density",
+#              "metal_density",
+# the following fields must be explicitly defined since
+# we want to add them to the "derived" group
+              "cooling_time",  
+              "temperature",
+              "pressure"
+     ];
+
+     padding = 0;
+ }
+
+ Group {
+   list = ["color", "derived"];
+
+   # fields that should be advected by hydro MUST be defined
+   # as color here.
+   # This is handled automatically in  EnzoMethodGrackle for
+   # the Grackle-specific fields (species). But, other
+   # passively advected fields must be added manually.
+   #
+   # As a test of the automatic field machinery, we comment
+   # out all of the grackle-specific-fields
+    color {
+       field_list = [];
+              #"HI_density",
+              #"HII_density",
+              #"HM_density",
+              #"HeI_density",
+              #"HeII_density",
+              #"HeIII_density",
+              #"H2I_density",
+              #"H2II_density",
+              #"DI_density",
+              #"DII_density",
+              #"HDI_density",
+              #"e_density",
+              #"metal_density"];
+    }
+
+#  derived fields grouping enables Enzo-E to ensure
+#  that all derived fields are updated prior to output
+#  In order for this to work, these MUST have their own
+#  compute classes. (these need to manually be added to the 
+    derived {
+        field_list = ["temperature",
+                      "pressure",
+                      "cooling_time"];
+    }
+ }
+
+ Initial {
+   list = ["grackle_test"];
+   grackle_test {
+      # See note in "Mesh" on running in < 3D
+
+      minimum_H_number_density = 0.0001;
+      maximum_H_number_density = 1000.0;
+      minimum_metallicity      = 1.0;
+      maximum_metallicity      = 1.0;
+      minimum_temperature      = 10.0;
+      maximum_temperature      = 1.0E8;
+
+      # Keep temperature constant with changing mu?
+      reset_energies           = 0;
+
+      dt = 100.0;
+   }
+ }
+
+ Mesh {
+     #   This test problem varies density, temperature,
+     #   and metallicity over x, y, and z dimensions 
+     #   respectively. For smaller problems you can 
+     #   run in 1 or 2 dimensions by changing the below
+     #   accordingly and setting the min/max values in
+     #   initialization to be the same. Currently cannot
+     #   pick which value goes in which dimension,
+     #   so 2D can only be density and temperature,
+     #   1D can only vary density.
+
+     #root_blocks = [ 4, 4];
+     #root_rank = 2;
+     #root_size = [128, 128];
+
+     # 3D test example:
+     root_blocks = [ 4, 4, 4];
+     root_rank = 3;
+     root_size = [32, 32, 32];
+
+ }
+
+ Method {
+      list = [ "grackle", "null"];
+
+     grackle {
+        courant = 0.40; # meaningless unless use_cooling_timestep = true;
+
+        data_file = "CloudyData_UVB=HM2012_shielded.h5";
+
+        with_radiative_cooling = 1;
+        primordial_chemistry   = 3;  # 1, 2, or 3
+        metal_cooling          = 1;  # 0 or 1 (off/on)
+        UVbackground           = 1;  # on or off
+        self_shielding_method  = 3;  # 0 - 3 (0 or 3 recommended)
+
+        HydrogenFractionByMass = 0.73;
+
+        # set this to true to limit the maximum timestep to the product of the
+        # minimum cooling/heating time and courant.
+        use_cooling_timestep = false; # default is false
+     }
+
+     # use this to limit maximum timestep if grackle::use_cooling_timestep is
+     # set to false and Grackle crashes due to max iteration limit - this is
+     # not needed generally in a real simulation as hydro timestep will
+     # be small (usually)
+     null { dt = 100.0; }
+
+ }
+
+ Output {
+     list = ["data"];
+
+     data {
+         field_list = [ "density",
+              "internal_energy",
+              "total_energy",
+              "velocity_x",
+              "velocity_y",
+              "velocity_z",
+              "HI_density",
+              "HII_density",
+              "HM_density",
+              "HeI_density",
+              "HeII_density",
+              "HeIII_density",
+              "H2I_density",
+              "H2II_density",
+              "DI_density",
+              "DII_density",
+              "HDI_density",
+              "e_density",
+              "metal_density",
+              "cooling_time",
+              "temperature",
+              "pressure"];
+
+         dir   = ["GRACKLE_TEST_%03d","cycle"];
+         name = [ "method_grackle-1-%03d.h5", "proc" ];
+         schedule {
+             var = "time";
+             step = 1000.0;
+             start = 0.0;
+         }
+         type = "data";
+     }
+
+ }
+

input/Grackle/method_grackle_cooling_dt.in

@@ -0,0 +1,24 @@
+# this file is used for a test where the cooling time is used to limit
+# the size of the timestep
+#
+# The automated test will also overwrite the entry in
+# Method:grackle:data_file parameter with a valid path
+
+include "input/Grackle/grackle.incl"
+
+Output { list = []; } # don't write any outputs
+
+Method {
+    list = [ "grackle" ]; # intentionally omit "null"
+    grackle {
+        courant = 0.40;
+        use_cooling_timestep = true;
+    }
+}
+
+Testing {
+    time_final  = [0.00201833123232718];
+    time_tolerance = 1.0e-4;
+}
+
+Stopping { cycle = 20; }

input/Grackle/method_grackle_general.in

@@ -0,0 +1,19 @@
+# this file is used for checking general consistency of grackle results
+# (without using a timestep based on the minimum cooling time)
+#
+# The automated test will also overwrite the entry in
+# Method:grackle:data_file parameter with a valid path
+
+include "input/Grackle/grackle.incl"
+
+Output {
+    data {
+        dir = ["GeneralGrackle-%06.2f", "time"];
+        schedule {
+            var = "time";
+            list = [500.0];
+        }
+    }
+}
+
+Stopping { time = 500.0; }

input/Grackle/ref_general_grackle-double.csv

@@ -0,0 +1,6 @@
+# {"code unit definitions": {"code_length": [3.086e+16, "cm"], "code_mass": [4.9157019637146824e+25, "g"], "code_density": [1.6726219e-24, "g/cm**3"], "code_specific_energy": [956277.4378779504, "erg/g"], "code_time": [31557600000000.0, "s"], "code_magnetic": [4.483279099741026e-09, "G"], "code_temperature": [1.0, "K"], "code_pressure": [1.5994905850705492e-18, "dyn/cm**2"], "code_velocity": [977.8943899409335, "cm/s"], "code_metallicity": [1.0, "dimensionless"]}, "field units": {"pressure": "dimensionless", "temperature": "code_temperature", "cooling_time": "dimensionless"}}
+#
+# name,min,min_xloc,min_yloc,min_zloc,max,max_xloc,max_yloc,max_zloc,mean,standard_deviation
+pressure,1.520195900675427e+02,7.812500000000000e-02,1.718750000000000e-01,1.562500000000000e-02,2.753947492701290e+06,7.812500000000000e-02,2.343750000000000e-01,1.562500000000000e-02,6.660831661417476e+04,3.557307908963668e+05
+temperature,3.330949646514823e+00,2.343750000000000e-01,1.718750000000000e-01,1.562500000000000e-02,6.376232820586985e+06,1.562500000000000e-02,2.343750000000000e-01,1.562500000000000e-02,2.417771759620777e+05,1.115430127734497e+06
+cooling_time,-6.450617403727239e+04,1.562500000000000e-02,2.343750000000000e-01,1.562500000000000e-02,3.866436114544078e+03,1.093750000000000e-01,1.093750000000000e-01,1.562500000000000e-02,-1.576307136350389e+03,8.485636211531526e+03

input/Grackle/ref_general_grackle-single.csv

@@ -0,0 +1,6 @@
+# {"code unit definitions": {"code_length": [3.086e+16, "cm"], "code_mass": [4.9157019637146824e+25, "g"], "code_density": [1.6726219e-24, "g/cm**3"], "code_specific_energy": [956277.4378779504, "erg/g"], "code_time": [31557600000000.0, "s"], "code_magnetic": [4.483279099741026e-09, "G"], "code_temperature": [1.0, "K"], "code_pressure": [1.5994905850705492e-18, "dyn/cm**2"], "code_velocity": [977.8943899409335, "cm/s"], "code_metallicity": [1.0, "dimensionless"]}, "field units": {"pressure": "dimensionless", "temperature": "code_temperature", "cooling_time": "dimensionless"}}
+#
+# name,min,min_xloc,min_yloc,min_zloc,max,max_xloc,max_yloc,max_zloc,mean,standard_deviation
+pressure,1.448596228495553e+02,7.812500000000000e-02,2.031250000000000e-01,1.562500000000000e-02,2.754013931249404e+06,7.812500000000000e-02,2.343750000000000e-01,1.562500000000000e-02,6.668474451175868e+04,3.557327613678277e+05
+temperature,3.444945335388184e+00,2.343750000000000e-01,2.031250000000000e-01,1.562500000000000e-02,6.376238000000000e+06,1.562500000000000e-02,2.343750000000000e-01,1.562500000000000e-02,2.417772680167406e+05,1.115433169502840e+06
+cooling_time,-6.450619921875000e+04,1.562500000000000e-02,2.343750000000000e-01,1.562500000000000e-02,1.856555786132812e+03,4.687500000000000e-02,7.812500000000000e-02,1.562500000000000e-02,-1.300657236445695e+03,8.072030883837461e+03