LES exp ClimaAtmos

Up to 0.14.11 CC
	modified:   src/parameterized_tendencies/les_sgs_models/smagorinsky_lilly.jl

modified:   post_processing/ci_plots.jl
 Untracked files:

.buildkite/longruns_gpu/pipeline.yml

@@ -11,7 +11,8 @@ env:
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   SLURM_KILL_BAD_EXIT: 1
   SLURM_GRES_FLAGS: "allow-task-sharing"
-  CONFIG_PATH: "config/longrun_configs"
+  GPU_CONFIG_PATH: "config/gpu_configs/"
+  MODEL_CONFIG_PATH: "config/model_configs/"
   CLIMAATMOS_GC_NSTEPS: 10
 
 timeout_in_minutes: 1440
@@ -22,6 +23,7 @@ steps:
     command:
       - echo "--- Instantiate examples"
       - julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'
+      - "julia --project=examples -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaCore.jl\", rev = \"as/covar-deriv\");'"
       - julia --project=examples -e 'using Pkg; Pkg.precompile()'
       - julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'
       - julia --project=examples -e 'using Pkg; Pkg.status()'
@@ -38,210 +40,59 @@ steps:
 
   - wait
 
-  - group: "helem 30 dycore"
+  - group: "les test"
     steps:
-
-      - label: ":computer: hydrostatic balance (ρe_tot)"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_sphere_hydrostatic_balance_rhoe"
-
-      - label: ":computer: dry baroclinic wave"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_dry_baroclinic_wave"
-
-      - label: ":computer: dry baroclinic wave high res"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_dry_baroclinic_wave_he60"
-
-      - label: ":computer: baroclinic wave equilmoist"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_moist_baroclinic_wave"
-
-      - label: ":computer: baroclinic wave equilmoist high res"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_moist_baroclinic_wave_he60"
-
-      - label: ":computer: dry held-suarez"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_dry_held_suarez"
-
-      - label: ":computer: held-suarez, equilmoist"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_moist_held_suarez"
-
-      - label: ":computer: held-suarez equilmoist + deep-atmosphere eqns"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_moist_held_suarez_deepatmos"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 12:00:00
-
-  - group: "helem 16 aquaplanet"
-    steps:
-
-      - label: ":computer: aquaplanet equilmoist allsky radiation + 0M microphysics"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_allsky_0M"
-
-      - label: ":computer: aquaplanet equilmoist allsky radiation + diagnostic edmf + 0M microphysics"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_allsky_diagedmf_0M"
 
-      - label: ":computer: aquaplanet equilmoist allsky radiation + prognostic edmf diffusion only + 0M microphysics"
+      - label: "dycoms: 2 GPUs"
         command:
-          - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
+          - >
+            srun --cpu-bind=threads --cpus-per-task=4
+            julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
+            --config_file config/model_configs/les_dycoms_box.yml
+            --job_id dycoms
+        artifact_paths: "dycoms_4gpu/output_active/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_allsky_progedmf_diffonly_0M"
-
-      - label: ":computer: aquaplanet equilmoist allsky radiation + 0M microphysics + earth topography"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_allsky_0M_earth"
-
-      - label: ":umbrella: aquaplanet equilmoist allsky radiation + 1M microphysics"
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 2
+          slurm_mem: 16G
+          slurm_time: 6:00:00
+      
+      - label: "bomex: 2 GPUs"
         command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
+          - >
+            srun --cpu-bind=threads --cpus-per-task=4
+            julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
+            --config_file config/model_configs/les_bomex_box.yml
+            --job_id bomex
+        artifact_paths: "bomex/output_active/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_allsky_1M"
-
-      - label: ":computer: aquaplanet equilmoist allsky radiation + time-varying insolation + 0M microphysics + slab ocean"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_allsky_tvinsol_0M_slabocean"
-
-  - group: "DYAMOND"
-
-    steps:
-
-      - label: ":computer: aquaplanet dyamond"
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 2
+          slurm_mem: 16G
+          slurm_time: 6:00:00
+
+      - label: "rico: 2 GPUs"
         command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
+          - >
+            srun --cpu-bind=threads --cpus-per-task=4
+            julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
+            --config_file config/model_configs/les_rico_box.yml
+            --job_id rico
+        artifact_paths: "rico/output_active/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "longrun_aquaplanet_dyamond"
-
-  - group: "atmos-only coupler runs"
-
-    steps:
-
-      - label: ":computer: amip target diagnostic edmf"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-        artifact_paths: "$$JOB_NAME/output_active/*"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
-          slurm_gpus: 1
-          slurm_time: 24:00:00
-        env:
-          CLIMACOMMS_DEVICE: "CUDA"
-          JOB_NAME: "amip_target_diagedmf"
-
-  # - group: "Experimental long runs"
-
-  #   steps:
-
-  #     TODO: uncomment when zalesak works
-  #     - label: ":computer: lim ARS zalesak baroclinic wave (ρe_tot) equilmoist high resolution"
-  #       command:
-  #         - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
-  #       artifact_paths: "$$JOB_NAME/output_active/*"
-  #       env:
-  #         CLIMACOMMS_CONTEXT: "MPI"
-  #       agents:
-  #         slurm_ntasks: 32
-  #         slurm_mem_per_cpu: 32GB
-  #         slurm_time: 24:00:00
-  #       env:
-  #         JOB_NAME: "longrun_zalesak_tracer_energy_bw_rhoe_equil_highres"
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 2
+          slurm_mem: 16G
+          slurm_time: 6:00:00

.buildkite/pipeline.yml

@@ -29,12 +29,14 @@ steps:
       - "julia --project -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate examples"
+      - "julia --project=examples -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaTimeSteppers.jl\", rev = \"gb/rosenbrock\");'"
       - "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=examples -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'"
       - "julia --project=examples -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate perf"
+      - "julia --project=perf -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaTimeSteppers.jl\", rev = \"gb/rosenbrock\");'"
       - "julia --project=perf -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=perf -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=perf -e 'using Pkg; Pkg.status()'"

config/default_configs/default_config.yml

@@ -202,6 +202,13 @@ perf_summary:
 viscous_sponge:
   help: "Viscous sponge [`true`, `false` (default)]"
   value: false
+smagorinsky_lilly:
+  value: false
+c_smag: 
+  value: 0.2
+job_id:
+  help: "Uniquely identifying string for a particular job"
+  value: ~
 tracer_upwinding:
   help: "Tracer upwinding mode [`none` (default), `first_order` , `third_order`, `boris_book`, `zalesak`]"
   value: none

config/model_configs/box_density_current_test.yml

@@ -1,16 +1,22 @@
+job_id: "box_density_current_test"
 dt_save_state_to_disk: "10secs"
 initial_condition: "DryDensityCurrentProfile"
 x_max: 51200.0
 z_elem: 45
-dt: "0.1secs"
-t_end: "10.0secs"
+dt: "0.3secs"
+t_end: "1000secs"
 discrete_hydrostatic_balance: true
-y_max: 6400.0
-y_elem: 15
+hyperdiff: "false"
+y_max: 51200.0
+y_elem: 45
 z_stretch: false
 x_elem: 45
 config: "box"
+hyperdiff: "true"
 z_max: 6400.0
+smagorinsky_lilly: true
+c_smag: 0.25
+netcdf_interpolation_num_points: [40, 40, 80]
 diagnostics:
-  - short_name: thetaa
-    period: 10secs
+  - short_name: [thetaa, wa, ua, ta, va, hfes, rhoa, rv, ha]
+    period: 30secs

config/model_configs/les_bomex_box.yml

@@ -0,0 +1,38 @@
+job_id: "les_bomex_box"
+initial_condition: "Bomex" 
+edmf_coriolis: "Bomex"
+subsidence: "Bomex" 
+ls_adv: "Bomex" 
+surface_setup: "Bomex"
+moist: "equil"
+config: "box" 
+hyperdiff: "true" 
+implicit_diffusion: false
+precip_model: "0M"
+vorticity_hyperdiffusion_coefficient: 0.2
+scalar_hyperdiffusion_coefficient: 0.2
+dt_cloud_fraction: "10mins"
+ode_algo: "SSPKnoth"
+smagorinsky_lilly: true
+c_smag: 0.18
+x_max: 3.2e3
+y_max: 3.2e3 
+z_max: 4.5e3
+x_elem: 8
+y_elem: 8
+z_elem: 100
+z_stretch: false
+dt: "0.10secs" 
+output_default_diagnostics: false
+t_end: "36000secs" 
+dt_save_state_to_disk: "30mins"
+rayleigh_sponge: true
+toml: [toml/bomex_box_rhoe.toml]
+restart_file: "/home/asridhar/Codes/ClimaAtmos.jl/restart/bomex/Micro0M_day0.0_Prod.hdf5"
+netcdf_interpolation_num_points: [40, 40, 80]
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl, kvis, hfes]
+    period: 10mins
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl, kvis, hfes]
+    reduction: average
+    period: 60mins