From aaceae8b70c85acd15ea56f6fc50254b677ef026 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Tue, 29 Aug 2023 14:49:14 -0600 Subject: [PATCH 001/165] added PopIII setup --- src/PopIII/CMakeLists.txt | 32 +++ src/PopIII/TurbDataReader.cpp | 119 ++++++++++ src/PopIII/TurbDataReader.hpp | 59 +++++ src/PopIII/ascent_actions.yaml | 30 +++ src/PopIII/perturbation.py | 311 +++++++++++++++++++++++++ src/PopIII/popiii.cpp | 404 +++++++++++++++++++++++++++++++++ src/PopIII/popiii.hpp | 22 ++ tests/PopIII.in | 94 ++++++++ 8 files changed, 1071 insertions(+) create mode 100644 src/PopIII/CMakeLists.txt create mode 100644 src/PopIII/TurbDataReader.cpp create mode 100644 src/PopIII/TurbDataReader.hpp create mode 100644 src/PopIII/ascent_actions.yaml create mode 100644 src/PopIII/perturbation.py create mode 100644 src/PopIII/popiii.cpp create mode 100644 src/PopIII/popiii.hpp create mode 100644 tests/PopIII.in diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt new file mode 100644 index 000000000..72ce0a0ff --- /dev/null +++ b/src/PopIII/CMakeLists.txt @@ -0,0 +1,32 @@ +if (AMReX_SPACEDIM EQUAL 3) + # Define a custom target that runs the Python script to produce the input perturbations file + + + set(microphysics_network_name "primordial_chem") #this will override network_name to primordial_chem for this directory only + setup_target_for_microphysics_compilation(${microphysics_network_name} "${CMAKE_CURRENT_BINARY_DIR}/") + + #use the BEFORE keyword so that these files get priority in compilation for targets in this directory + #this is critical to ensure the correct Microphysics files are linked to primordial chem target + include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H") + + add_executable(popiii popiii.cpp TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) + target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM + + if(AMReX_GPU_BACKEND MATCHES "CUDA") + setup_target_for_cuda_compilation(star_cluster) + endif() + + execute_process( + COMMAND Python3::Interpreter -c "h5py" + RESULT_VARIABLE EXIT_CODE + OUTPUT_QUIET + ) + + add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + add_test(NAME PopIII COMMAND popiii popiii.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP StarCluster_fixture) + set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED StarCluster_fixture) + + # AMR test only works on Setonix because Gadi and avatar do not have enough memory per GPU + # add_test(NAME PopIIIAMR COMMAND popiii popiii_AMR.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) +endif() diff --git a/src/PopIII/TurbDataReader.cpp b/src/PopIII/TurbDataReader.cpp new file mode 100644 index 000000000..6834c17a1 --- /dev/null +++ b/src/PopIII/TurbDataReader.cpp @@ -0,0 +1,119 @@ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file TurbDataReader.cpp +/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. +/// + +#include "TurbDataReader.hpp" +#include "AMReX_Arena.H" +#include "AMReX_BLassert.H" +#include "AMReX_Print.H" +#include "AMReX_TableData.H" +#include + +auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D +{ + // open dataset + hid_t dset_id = 0; + dset_id = H5Dopen2(file_id, dataset_name, H5P_DEFAULT); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dset_id != -1, "Can't open table!"); + + // get dimensions + hid_t const dspace = H5Dget_space(dset_id); + const int ndims = H5Sget_simple_extent_ndims(dspace); + std::vector dims(ndims); + H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); + + uint64_t data_size = 1; + for (int idim = 0; idim < ndims; ++idim) { + data_size *= dims[idim]; + } + + // allocate array for dataset storage + auto *temp_data = new double[data_size]; + + // read dataset + herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); + + // close dataset + status = H5Dclose(dset_id); + + // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 + // tables use row-major (C-order) indexing! + amrex::GpuArray const lo{0, 0, 0}; + amrex::GpuArray const hi{static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])}; + auto table = amrex::Table3D(temp_data, lo, hi); + return table; +} + +void initialize_turbdata(turb_data &data, std::string &data_file) +{ + amrex::Print() << "Initializing turbulence data...\n"; + amrex::Print() << fmt::format("data_file: {}.\n", data_file); + + herr_t status = 0; + herr_t const h5_error = -1; + + // open file + hid_t file_id = 0; + file_id = H5Fopen(data_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(file_id != h5_error, "Failed to open data file!"); + + data.dvx = read_dataset(file_id, "pertx"); + data.dvy = read_dataset(file_id, "perty"); + data.dvz = read_dataset(file_id, "pertz"); + + // close file + status = H5Fclose(file_id); +} + +auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData +{ + amrex::Array tlo{in_t.begin[0], in_t.begin[1], in_t.begin[2]}; + amrex::Array thi{in_t.end[0] - 1, in_t.end[1] - 1, in_t.end[2] - 1}; + amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); + auto h_table = tableData.table(); + + amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; + + // fill tableData + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + for (int k = tlo[2]; k <= thi[2]; ++k) { + h_table(i, j, k) = in_t(i, j, k); + } + } + } + + return tableData; +} + +auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real +{ + amrex::Array tlo = dvx.lo(); + amrex::Array thi = dvx.hi(); + auto const &dvx_table = dvx.const_table(); + auto const &dvy_table = dvy.const_table(); + auto const &dvz_table = dvz.const_table(); + + // compute rms power + amrex::Real rms_sq = 0; + amrex::Long N = 0; + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + for (int k = tlo[2]; k <= thi[2]; ++k) { + amrex::Real const vx = dvx_table(i, j, k); + amrex::Real const vy = dvy_table(i, j, k); + amrex::Real const vz = dvz_table(i, j, k); + rms_sq += vx * vx + vy * vy + vz * vz; + ++N; + } + } + } + rms_sq /= static_cast(N); + return std::sqrt(rms_sq); +} diff --git a/src/PopIII/TurbDataReader.hpp b/src/PopIII/TurbDataReader.hpp new file mode 100644 index 000000000..b127cc92a --- /dev/null +++ b/src/PopIII/TurbDataReader.hpp @@ -0,0 +1,59 @@ +#ifndef TURBDATAREADER_HPP_ // NOLINT +#define TURBDATAREADER_HPP_ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file TurbDataReader.hpp +/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. +/// + +#include +#include +#include +#include +#include + +#include "fmt/core.h" +#include +#include +#include + +#include "AMReX.H" +#include "AMReX_Array.H" +#include "AMReX_BLassert.H" +#include "AMReX_TableData.H" + +/* HDF5 definitions */ + +#define HDF5_FILE_I4 H5T_STD_I32BE +#define HDF5_FILE_I8 H5T_STD_I64BE +#define HDF5_FILE_R4 H5T_IEEE_F32BE +#define HDF5_FILE_R8 H5T_IEEE_F64BE +#define HDF5_FILE_B8 H5T_STD_B8BE + +#define HDF5_I4 H5T_NATIVE_INT +#define HDF5_I8 H5T_NATIVE_LLONG +#define HDF5_R4 H5T_NATIVE_FLOAT +#define HDF5_R8 H5T_NATIVE_DOUBLE +#define HDF5_R16 H5T_NATIVE_LDOUBLE + +// Cooling table storage + +using turb_data = struct turb_data { + // values + amrex::Table3D dvx; + amrex::Table3D dvy; + amrex::Table3D dvz; +}; + +void initialize_turbdata(turb_data &data, std::string &data_file); + +auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D; + +auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData; + +auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real; + +#endif // TURBDATAREADER_HPP_ diff --git a/src/PopIII/ascent_actions.yaml b/src/PopIII/ascent_actions.yaml new file mode 100644 index 000000000..6515c4b84 --- /dev/null +++ b/src/PopIII/ascent_actions.yaml @@ -0,0 +1,30 @@ +- + action: "add_pipelines" + pipelines: + pl1: + f2: + type: "slice" + params: + point: + x: 0. + y: 0. + z: 0. + normal: + x: 0.0 + y: 0.0 + z: 1.0 +- + action: "add_scenes" + scenes: + s1: + plots: + p1: + type: "pseudocolor" + field: "log_density" + pipeline: "pl1" + renders: + r1: + image_prefix: "dens%05d" + annotations: "true" + camera: + zoom: 1.5 diff --git a/src/PopIII/perturbation.py b/src/PopIII/perturbation.py new file mode 100644 index 000000000..d6d53e640 --- /dev/null +++ b/src/PopIII/perturbation.py @@ -0,0 +1,311 @@ +import h5py +import numpy as np +import numpy.fft +from math import * +from optparse import OptionParser +import sys + + +def init_perturbations(n, kmin, kmax, dtype): + kx = np.zeros(n, dtype=dtype) + ky = np.zeros(n, dtype=dtype) + kz = np.zeros(n, dtype=dtype) + # perform fft k-ordering convention shifts + for j in range(0,n[1]): + for k in range(0,n[2]): + kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) + for i in range(0,n[0]): + for k in range(0,n[2]): + ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) + for i in range(0,n[0]): + for j in range(0,n[1]): + kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) + + kx = np.array(kx, dtype=dtype) + ky = np.array(ky, dtype=dtype) + kz = np.array(kz, dtype=dtype) + k = np.sqrt(np.array(kx**2+ky**2+kz**2, dtype=dtype)) + + # only use the positive frequencies + inds = np.where(np.logical_and(k**2 >= kmin**2, k**2 < (kmax+1)**2)) + nr = len(inds[0]) + + phasex = np.zeros(n, dtype=dtype) + phasex[inds] = 2.*pi*np.random.uniform(size=nr) + fx = np.zeros(n, dtype=dtype) + fx[inds] = np.random.normal(size=nr) + + phasey = np.zeros(n, dtype=dtype) + phasey[inds] = 2.*pi*np.random.uniform(size=nr) + fy = np.zeros(n, dtype=dtype) + fy[inds] = np.random.normal(size=nr) + + phasez = np.zeros(n, dtype=dtype) + phasez[inds] = 2.*pi*np.random.uniform(size=nr) + fz = np.zeros(n, dtype=dtype) + fz[inds] = np.random.normal(size=nr) + + # rescale perturbation amplitude so that low number statistics + # at low k do not throw off the desired power law scaling. + for i in range(kmin, kmax+1): + slice_inds = np.where(np.logical_and(k >= i, k < i+1)) + rescale = sqrt(np.sum(np.abs(fx[slice_inds])**2 + np.abs(fy[slice_inds])**2 + np.abs(fz[slice_inds])**2)) + fx[slice_inds] = fx[slice_inds]/rescale + fy[slice_inds] = fy[slice_inds]/rescale + fz[slice_inds] = fz[slice_inds]/rescale + + # set the power law behavior + # wave number bins + fx[inds] = fx[inds]*k[inds]**-(0.5*alpha) + fy[inds] = fy[inds]*k[inds]**-(0.5*alpha) + fz[inds] = fz[inds]*k[inds]**-(0.5*alpha) + + # add in phases + fx = np.cos(phasex)*fx + 1j*np.sin(phasex)*fx + fy = np.cos(phasey)*fy + 1j*np.sin(phasey)*fy + fz = np.cos(phasez)*fz + 1j*np.sin(phasez)*fz + + return fx, fy, fz, kx, ky, kz + + +def normalize(fx, fy, fz): + norm = np.sqrt(np.sum(fx**2 + fy**2 + fz**2)/np.product(n)) + fx = fx/norm + fy = fy/norm + fz = fz/norm + return fx, fy, fz + + +def make_perturbations(n, kmin, kmax, f_solenoidal): + fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) + if f_solenoidal != None: + k2 = kx**2+ky**2+kz**2 + # solenoidal part + fxs = 0.; fys =0.; fzs = 0. + if f_solenoidal != 0.0: + fxs = np.real(fx - kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fys = np.real(fy - ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fzs = np.real(fz - kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + ind = np.where(k2 == 0) + fxs[ind] = 0.; fys[ind] = 0.; fzs[ind] = 0. + # need to normalize this before applying relative weighting of solenoidal / compressive components + norm = np.sqrt(np.sum(fxs**2+fys**2+fzs**2)) + fxs = fxs/norm + fys = fys/norm + fzs = fzs/norm + # compressive part + # get a different random cube for the compressive part + # so that we can target the RMS solenoidal fraction, + # instead of setting a constant solenoidal fraction everywhere. + fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) + fxc = 0.; fyc =0.; fzc = 0. + if f_solenoidal != 1.0: + fxc = np.real(kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fyc = np.real(ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fzc = np.real(kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + ind = np.where(k2 == 0) + fxc[ind] = 0.; fyc[ind] = 0.; fzc[ind] = 0. + # need to normalize this before applying relative weighting of solenoidal / compressive components + norm = np.sqrt(np.sum(fxc**2+fyc**2+fzc**2)) + fxc = fxc/norm + fyc = fyc/norm + fzc = fzc/norm + # back to real space + pertx = np.real(np.fft.ifftn(f_solenoidal*fxs + (1.-f_solenoidal)*fxc)) + perty = np.real(np.fft.ifftn(f_solenoidal*fys + (1.-f_solenoidal)*fyc)) + pertz = np.real(np.fft.ifftn(f_solenoidal*fzs + (1.-f_solenoidal)*fzc)) + else: + # just convert to real space + pertx = np.real(np.fft.ifftn(fx)) + perty = np.real(np.fft.ifftn(fy)) + pertz = np.real(np.fft.ifftn(fz)) + + # subtract off COM (assuming uniform density) + pertx = pertx-np.average(pertx) + perty = perty-np.average(perty) + pertz = pertz-np.average(pertz) + # scale RMS of perturbation cube to unity + pertx, perty, pertz = normalize(pertx, perty, pertz) + return pertx, perty, pertz + + +def cut_sphere(pertx, perty, pertz, rad): + # Make radial array + x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] + x = x - (n[0]-1)/2. + y = y - (n[1]-1)/2. + z = z - (n[2]-1)/2. + r2 = x**2+y**2+z**2 + # Get range of indices we want to set to zero, and those we want to keep + idx0 = r2 > (rad*n[0]/2.0)**2 + idx1 = np.logical_not(idx0) + # Zero outside the desired region + pertx[idx0] = 0.0 + perty[idx0] = 0.0 + pertz[idx0] = 0.0 + # Recompute COM velocity, and renormalize + pertx[idx1] = pertx[idx1]-np.average(pertx[idx1]) + perty[idx1] = perty[idx1]-np.average(perty[idx1]) + pertz[idx1] = pertz[idx1]-np.average(pertz[idx1]) + pertx, perty, pertz = normalize(pertx, perty, pertz) + return pertx, perty, pertz + + +def get_erot_ke_ratio(pertx, perty, pertz, rad=-1.0): + x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] + x = x - (n[0]-1)/2. + y = y - (n[1]-1)/2. + z = z - (n[2]-1)/2. + r2 = x**2+y**2+z**2 + if rad > 0: + idx0 = r2 > (rad*n[0]/2.0)**2 + r2[idx0] = 0.0 + erot_ke_ratio = (np.sum(y*pertz-z*perty)**2 + + np.sum(z*pertx-x*pertz)**2 + + np.sum(x*perty-y*pertx)**2)/(np.sum(r2)*np.product(n)) + return erot_ke_ratio + + +def plot_spectrum1D(pertx, perty, pertz): + # plot the 1D power to check the scaling. + fx = np.abs(np.fft.fftn(pertx)) + fy = np.abs(np.fft.fftn(perty)) + fz = np.abs(np.fft.fftn(pertz)) + fx = np.abs(fx) + fy = np.abs(fy) + fz = np.abs(fz) + kx = np.zeros(n, dtype=dtype) + ky = np.zeros(n, dtype=dtype) + kz = np.zeros(n, dtype=dtype) + # perform fft k-ordering convention shifts + for j in range(0,n[1]): + for k in range(0,n[2]): + kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) + for i in range(0,n[0]): + for k in range(0,n[2]): + ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) + for i in range(0,n[0]): + for j in range(0,n[1]): + kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) + k = np.sqrt(np.array(kx**2+ky**2+kz**2,dtype=dtype)) + k1d = [] + power = [] + for i in range(kmin,kmax+1): + slice_inds = np.where(np.logical_and(k >= i, k < i+1)) + k1d.append(i+0.5) + power.append(np.sum(fx[slice_inds]**2 + fy[slice_inds]**2 + fz[slice_inds]**2)) + print(i,power[-1]) + import matplotlib.pyplot as plt + plt.loglog(k1d, power) + plt.show() + + +################### +# input parameters, read from command line +################### +parser = OptionParser() +parser.add_option('--kmin', dest='kmin', + help='minimum wavenumber.', + default=-1) +parser.add_option('--kmax', dest='kmax', + help='maximum wavenumber.', + default=-1) +parser.add_option('--size', dest='size', + help='size of each direction of data cube. default=256', + default=256) +parser.add_option('--alpha', dest='alpha', + help='negative of power law slope. (Power ~ k^-alpha) '+ + 'supersonic turbulence is near alpha=2. '+ + 'driving over a narrow band of two modes is often done with alpha=0', + default = None) +parser.add_option('--seed', dest='seed', + help='seed for random # generation. default=0', + default = 0) +parser.add_option('--f_solenoidal', dest='f_solenoidal', + help='volume RMS fraction of solenoidal component of the perturbations relative to the total. ' + + 'If --f_solenoidal=None, the motions are purely random. For low wave numbers ' + + 'the relative imporance of solenoidal to compressive may be sensitive to the ' + + 'choice of radom seed. It has been suggested (Federrath 2008) that ' + + 'f_solenoidal=2/3 is the most natural driving mode and this is currently' + + 'the suggested best-practice.', + default = -1) +parser.add_option('--sphererad', dest='rad', + help='if set, perturbations are set to zero outside spherical region, '+ + 'and the perturbation field is shifted and renormalized to keep the '+ + 'center of mass velocity at zero and the variance at unity; the '+ + 'spherical region cut out is centered at the center of the perturbation '+ + 'cube, and has a radius given by the value of this parameter, with sphererad = '+ + '1 corresponding to the spherical region going all the way to the edge of the '+ + 'perturbation cube', + default = -1.0) + +(options, args) = parser.parse_args() + +# size of the data domain +n = [int(options.size), int(options.size), int(options.size)] +# range of perturbation length scale in units of the smallest side of the domain +kmin = int(options.kmin) +kmax = int(options.kmax) +print(kmin, kmax) +if kmin > kmax or kmin < 0 or kmax < 0: + print("kmin must be < kmax, with kmin > 0, kmax > 0. See --help.") + sys.exit(0) +if kmax > floor(np.min(n))/2: + print("kmax must be <= floor(size/2). See --help.") + sys.exit(0) +f_solenoidal = options.f_solenoidal +if f_solenoidal == "None" or f_solenoidal == "none": + f_solenoidal = None +else: + f_solenoidal = float(options.f_solenoidal) + if f_solenoidal > 1. or f_solenoidal < 0.: + print("You must choose f_solenoidal. See --help.") + sys.exit(0) +alpha = options.alpha +if alpha==None: + print("You must choose a power law slope, alpha. See --help.") + sys.exit(0) +alpha = float(options.alpha) +if alpha < 0.: + print("alpha is less than zero. That's probably not what you want. See --help.") + sys.exit(0) +seed = int(options.seed) +# data precision +dtype = np.float64 +# ratio of solenoidal to compressive components +if options.f_solenoidal=="None" or options.f_solenoidal==None: + f_solenoidal = None +else: + f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) +rad = float(options.rad) +if rad > 1: + raise ValueError('sphererad is '+options.rad+', must be from 0 to 1') + +################### +# begin computation +################### + +np.random.seed(seed=seed) +pertx, perty, pertz = make_perturbations(n, kmin, kmax, f_solenoidal) +if rad > 0: + pertx, perty, pertz = cut_sphere(pertx, perty, pertz, rad) +erot_ke_ratio = get_erot_ke_ratio(pertx, perty, pertz, rad) +print("erot_ke_ratio = ", erot_ke_ratio) + +# hdf5 output +f = h5py.File('zdrv.hdf5', 'w') +ds = f['/'].create_dataset('pertx', n, dtype=np.float64) +ds[:] = pertx +ds = f['/'].create_dataset('perty', n, dtype=np.float64) +ds[:] = perty +ds = f['/'].create_dataset('pertz', n, dtype=np.float64) +ds[:] = pertz +f['/'].attrs['kmin'] = kmin +f['/'].attrs['kmax'] = kmax +f['/'].attrs['alpha'] = alpha +if f_solenoidal!=None: f['/'].attrs['f_solenoidal'] = f_solenoidal +if rad > 0: f['/'].attrs['sphererad'] = rad +f['/'].attrs['erot_ke_ratio'] = erot_ke_ratio +f['/'].attrs['seed'] = seed +f.close() + diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp new file mode 100644 index 000000000..eff4d5e79 --- /dev/null +++ b/src/PopIII/popiii.cpp @@ -0,0 +1,404 @@ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file popiii.cpp +/// \brief Defines a test problem for Pop III star formation. +/// +#include +#include +#include + +#include "AMReX.H" +#include "AMReX_Arena.H" +#include "AMReX_BC_TYPES.H" +#include "AMReX_BLassert.H" +#include "AMReX_Config.H" +#include "AMReX_FabArrayUtility.H" +#include "AMReX_MultiFab.H" +#include "AMReX_ParallelDescriptor.H" +#include "AMReX_ParmParse.H" +#include "AMReX_Print.H" +#include "AMReX_SPACE.H" +#include "AMReX_TableData.H" + +#include "RadhydroSimulation.hpp" +#include "SimulationData.hpp" +#include "hydro_system.hpp" +#include "radiation_system.hpp" +#include "popiii.hpp" +#include "TurbDataReader.hpp" + +#include "actual_eos_data.H" +#include "burn_type.H" +#include "eos.H" +#include "extern_parameters.H" +#include "network.H" + +using amrex::Real; + +struct PopIII { +}; + +template <> struct HydroSystem_Traits { + static constexpr bool reconstruct_eint = false; +}; + +template <> struct Physics_Traits { + // cell-centred + static constexpr bool is_hydro_enabled = true; + static constexpr bool is_chemistry_enabled = false; // in the future, this could point to microphysics, and set to true + static constexpr int numMassScalars = NumSpec; // number of chemical species + static constexpr int numPassiveScalars = numMassScalars + 0; // we only have mass scalars + static constexpr bool is_radiation_enabled = false; + // face-centred + static constexpr bool is_mhd_enabled = false; +}; + + +template <> struct SimulationData { + // real-space perturbation fields + amrex::TableData dvx; + amrex::TableData dvy; + amrex::TableData dvz; + amrex::Real dv_rms_generated{}; + amrex::Real dv_rms_target{}; + amrex::Real rescale_factor{}; + + // cloud parameters + amrex::Real R_sphere{}; + amrex::Real rho_sphere{}; + + amrex::Real small_temp; + amrex::Real small_dens; + amrex::Real temperature; + amrex::Real primary_species_1; + amrex::Real primary_species_2; + amrex::Real primary_species_3; + amrex::Real primary_species_4; + amrex::Real primary_species_5; + amrex::Real primary_species_6; + amrex::Real primary_species_7; + amrex::Real primary_species_8; + amrex::Real primary_species_9; + amrex::Real primary_species_10; + amrex::Real primary_species_11; + amrex::Real primary_species_12; + amrex::Real primary_species_13; + amrex::Real primary_species_14; + + +}; + +template <> void RadhydroSimulation::preCalculateInitialConditions() +{ + + // initialize microphysics routines + init_extern_parameters(); + + // parmparse species and temperature + amrex::ParmParse pp("primordial_chem"); + userData_.small_temp = 1e1; + pp.query("small_temp", userData_.small_temp); + + userData_.small_dens = 1e-60; + pp.query("small_dens", userData_.small_dens); + + userData_.temperature = 1e1; + pp.query("temperature", userData_.temperature); + + userData_.primary_species_1 = 1.0e0_rt; + userData_.primary_species_2 = 0.0e0_rt; + userData_.primary_species_3 = 0.0e0_rt; + userData_.primary_species_4 = 0.0e0_rt; + userData_.primary_species_5 = 0.0e0_rt; + userData_.primary_species_6 = 0.0e0_rt; + userData_.primary_species_7 = 0.0e0_rt; + userData_.primary_species_8 = 0.0e0_rt; + userData_.primary_species_9 = 0.0e0_rt; + userData_.primary_species_10 = 0.0e0_rt; + userData_.primary_species_11 = 0.0e0_rt; + userData_.primary_species_12 = 0.0e0_rt; + userData_.primary_species_13 = 0.0e0_rt; + userData_.primary_species_14 = 0.0e0_rt; + + pp.query("primary_species_1", userData_.primary_species_1); + pp.query("primary_species_2", userData_.primary_species_2); + pp.query("primary_species_3", userData_.primary_species_3); + pp.query("primary_species_4", userData_.primary_species_4); + pp.query("primary_species_5", userData_.primary_species_5); + pp.query("primary_species_6", userData_.primary_species_6); + pp.query("primary_species_7", userData_.primary_species_7); + pp.query("primary_species_8", userData_.primary_species_8); + pp.query("primary_species_9", userData_.primary_species_9); + pp.query("primary_species_10", userData_.primary_species_10); + pp.query("primary_species_11", userData_.primary_species_11); + pp.query("primary_species_12", userData_.primary_species_12); + pp.query("primary_species_13", userData_.primary_species_13); + pp.query("primary_species_14", userData_.primary_species_14); + + eos_init(userData_.small_temp, userData_.small_dens); + network_init(); + + + static bool isSamplingDone = false; + if (!isSamplingDone) { + // read perturbations from file + turb_data turbData; + amrex::ParmParse pp("perturb"); + std::string turbdata_filename; + pp.query("filename", turbdata_filename); + initialize_turbdata(turbData, turbdata_filename); + + // copy to pinned memory + auto pinned_dvx = get_tabledata(turbData.dvx); + auto pinned_dvy = get_tabledata(turbData.dvy); + auto pinned_dvz = get_tabledata(turbData.dvz); + + // compute normalisation + userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); + amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; + + const Real R_sphere = userData_.R_sphere; + const Real rho_sph = userData_.rho_sphere; + + const Real rms_dv_target = 1.8e5; + const Real rms_Mach_target = 1.0; + const Real rms_dv_actual = userData_.dv_rms_generated; + userData_.rescale_factor = rms_dv_target / rms_dv_actual; + amrex::Print() << "rms Mach target = " << rms_Mach_target << "\n"; + + // copy to GPU + userData_.dvx.resize(pinned_dvx.lo(), pinned_dvx.hi()); + userData_.dvx.copy(pinned_dvx); + + userData_.dvy.resize(pinned_dvy.lo(), pinned_dvy.hi()); + userData_.dvy.copy(pinned_dvy); + + userData_.dvz.resize(pinned_dvz.lo(), pinned_dvz.hi()); + userData_.dvz.copy(pinned_dvz); + + isSamplingDone = true; + } +} + +template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +{ + // set initial conditions + amrex::GpuArray const dx = grid_elem.dx_; + amrex::GpuArray prob_lo = grid_elem.prob_lo_; + amrex::GpuArray prob_hi = grid_elem.prob_hi_; + const amrex::Box &indexRange = grid_elem.indexRange_; + const amrex::Array4 &state_cc = grid_elem.array_; + + burn_t state; + + std::array numdens = {-1.0}; + + for (int n = 1; n <= NumSpec; ++n) { + switch (n) { + + case 1: + numdens[n - 1] = userData_.primary_species_1; + break; + case 2: + numdens[n - 1] = userData_.primary_species_2; + break; + case 3: + numdens[n - 1] = userData_.primary_species_3; + break; + case 4: + numdens[n - 1] = userData_.primary_species_4; + break; + case 5: + numdens[n - 1] = userData_.primary_species_5; + break; + case 6: + numdens[n - 1] = userData_.primary_species_6; + break; + case 7: + numdens[n - 1] = userData_.primary_species_7; + break; + case 8: + numdens[n - 1] = userData_.primary_species_8; + break; + case 9: + numdens[n - 1] = userData_.primary_species_9; + break; + case 10: + numdens[n - 1] = userData_.primary_species_10; + break; + case 11: + numdens[n - 1] = userData_.primary_species_11; + break; + case 12: + numdens[n - 1] = userData_.primary_species_12; + break; + case 13: + numdens[n - 1] = userData_.primary_species_13; + break; + case 14: + numdens[n - 1] = userData_.primary_species_14; + break; + + default: + amrex::Abort("Cannot initialize number density for chem specie"); + break; + } + } + + state.T = userData_.temperature; + + // find the density in g/cm^3 + Real rhotot = 0.0_rt; + for (int n = 0; n < NumSpec; ++n) { + state.xn[n] = numdens[n]; + rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS + } + + state.rho = rhotot; + + // normalize -- just in case + + std::array mfracs = {-1.0}; + Real msum = 0.0_rt; + for (int n = 0; n < NumSpec; ++n) { + mfracs[n] = state.xn[n] * spmasses[n] / rhotot; + msum += mfracs[n]; + } + + for (int n = 0; n < NumSpec; ++n) { + mfracs[n] /= msum; + // use the normalized mass fractions to obtain the corresponding number densities + state.xn[n] = mfracs[n] * rhotot / spmasses[n]; + } + + // call the EOS to set initial internal energy e + eos(eos_input_rt, state); + + amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); + amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); + amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); + + // cloud parameters + const double R_sphere = userData_.R_sphere; + + auto const &dvx = userData_.dvx.const_table(); + auto const &dvy = userData_.dvy.const_table(); + auto const &dvz = userData_.dvz.const_table(); + + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + amrex::Real const x = prob_lo[0] + (i + static_cast(0.5)) * dx[0]; + amrex::Real const y = prob_lo[1] + (j + static_cast(0.5)) * dx[1]; + amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx[2]; + amrex::Real const r = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2) + std::pow(z - z0, 2)); + + amrex::Real rho = 0; + + if (r <= R_sphere) { + rho = rhotot; + } else { + rho = 0.01*rhotot; + } + + amrex::Real e = rho * state.e; + + double vx = dvx(i, j, k); + double vy = dvy(i, j, k); + double vz = dvz(i, j, k); + + state_cc(i, j, k, HydroSystem::density_index) = rho; + state_cc(i, j, k, HydroSystem::x1Momentum_index) = rho * vx; + state_cc(i, j, k, HydroSystem::x2Momentum_index) = rho * vy; + state_cc(i, j, k, HydroSystem::x3Momentum_index) = rho * vz; + state_cc(i, j, k, HydroSystem::internalEnergy_index) = e; + + Real const Egas = RadSystem::ComputeEgasFromEint(rho, rho*vx, rho*vy, rho*vz, e); + state_cc(i, j, k, HydroSystem::energy_index) = Egas; + + for (int nn = 0; nn < NumSpec; ++nn) { + state_cc(i, j, k, HydroSystem::scalar0_index + nn) = + mfracs[nn] * rho; // we use partial densities and not mass fractions + } + + }); +} + +template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) +{ + // refine on Jeans length + const int N_cells = 32; // inverse of the 'Jeans number' [Truelove et al. (1997)] + const amrex::Real cs = quokka::EOS_Traits::cs_isothermal; + const amrex::Real dx = geom[lev].CellSizeArray()[0]; + const amrex::Real G = Gconst_; + + auto const &state = state_new_cc_[lev].const_arrays(); + auto tag = tags.arrays(); + + amrex::ParallelFor(tags, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); + + if (l_Jeans < (N_cells * dx)) { + tag[bx](i, j, k) = amrex::TagBox::SET; + } + }); +} + +template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const +{ + // compute derived variables and save in 'mf' + if (dname == "log_density") { + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + output[bx](i, j, k, ncomp) = std::log10(rho); + }); + } +} + +auto problem_main() -> int +{ + // read problem parameters + amrex::ParmParse const pp("perturb"); + + // cloud radius + Real R_sphere{}; + pp.query("cloud_radius", R_sphere); + + // cloud density + Real rho_sphere{}; + pp.query("cloud_density", rho_sphere); + + + // boundary conditions + const int ncomp_cc = Physics_Indices::nvarTotal_cc; + amrex::Vector BCs_cc(ncomp_cc); + for (int n = 0; n < ncomp_cc; ++n) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + BCs_cc[n].setLo(i, amrex::BCType::foextrap); + BCs_cc[n].setHi(i, amrex::BCType::foextrap); + } + } + + // Problem initialization + RadhydroSimulation sim(BCs_cc); + sim.doPoissonSolve_ = 1; // enable self-gravity + sim.densityFloor_ = 0.01; + + sim.userData_.R_sphere = R_sphere; + sim.userData_.rho_sphere = rho_sphere; + + // initialize + sim.setInitialConditions(); + + // evolve + sim.evolve(); + + int const status = 0; + return status; +} diff --git a/src/PopIII/popiii.hpp b/src/PopIII/popiii.hpp new file mode 100644 index 000000000..6fb4cd7ce --- /dev/null +++ b/src/PopIII/popiii.hpp @@ -0,0 +1,22 @@ +#ifndef POPIII_HPP_ // NOLINT +#define POPIII_HPP_ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file star_cluster.hpp +/// \brief Defines a test problem for Pop III star formation. +/// + +// external headers +#include + +// internal headers +#include "hydro_system.hpp" +#include "interpolate.hpp" + +// function definitions +auto problem_main() -> int; + +#endif // POPIII_HPP_ diff --git a/tests/PopIII.in b/tests/PopIII.in new file mode 100644 index 000000000..00b06ac89 --- /dev/null +++ b/tests/PopIII.in @@ -0,0 +1,94 @@ +# ***************************************************************** +# Problem size and geometry +# ***************************************************************** +geometry.prob_lo = -3.703e18 -3.703e18 -3.703e18 +geometry.prob_hi = 3.703e18 3.703e18 3.703e18 +geometry.is_periodic = 0 0 0 + +# ***************************************************************** +# VERBOSITY +# ***************************************************************** +amr.v = 1 # verbosity in Amr + +# ***************************************************************** +# Resolution and refinement +# ***************************************************************** +amr.n_cell = 128 128 128 +amr.max_level = 0 # number of levels = max_level + 1 +amr.blocking_factor = 16 # grid size must be divisible by this +amr.max_grid_size = 64 # at least 128 for GPUs +amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells +amr.grid_eff = 0.7 # default + +hydro.reconstruction_order = 3 # PPM +cfl = 0.2 +max_timesteps = 300 +stop_time = 1e16 + +do_reflux = 1 +do_subcycle = 0 + +ascent_interval = 10 +plotfile_interval = 20 +checkpoint_interval = 100 + +perturb.cloud_radius = 3.086e18 + +# in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' +# and put it in quokka/tests/ +perturb.filename = "zdrv.hdf5" + +derived_vars = log_density + +amrex.throw_exception = 0 +amrex.signal_handling = 1 + +primordial_chem.enabled = 1 +primordial_chem.temperature = 1e2 + +primordial_chem.small_temp = 1.e1 +primordial_chem.small_dens = 1.e-60 +primordial_chem.max_density_allowed = 3e-6 + +#ntot Tgas E H- D- H HE H2 HD D H+ HE+ H2+ D+ HD+ HE++ +primordial_chem.primary_species_1 = 1e-4 +primordial_chem.primary_species_2 = 1e-4 +primordial_chem.primary_species_3 = 1e0 +primordial_chem.primary_species_4 = 1e-40 +primordial_chem.primary_species_5 = 1e-40 +primordial_chem.primary_species_6 = 1e-40 +primordial_chem.primary_species_7 = 1e-40 +primordial_chem.primary_species_8 = 1e-40 +primordial_chem.primary_species_9 = 1e-6 +primordial_chem.primary_species_10 = 1e-40 +primordial_chem.primary_species_11 = 1e-40 +primordial_chem.primary_species_12 = 1e-40 +primordial_chem.primary_species_13 = 1e-40 +primordial_chem.primary_species_14 = 0.0775e0 + + +# integrator runtime parameters +# are we using primordial chemistry? then we use number densities +integrator.use_number_densities = 1 +# we do not want to subtract the internal energy +integrator.subtract_internal_energy = 0 +# we do not want to clip species between 0 and 1 +integrator.do_species_clip = 0 +# minimum positive value of number densities +integrator.SMALL_X_SAFE = 1e-100 +integrator.burner_verbose = 0 + +# do you want to use the jacobian calculated in a previous step? +integrator.use_jacobian_caching = 0 +# integration will fail if the number density > X_reject_buffer*atol +integrator.X_reject_buffer = 1e100 +# Set which jacobian to use +# 1 = analytic jacobian +# 2 = numerical jacobian +integrator.jacobian = 1 + +# do you want to normalize abundances within VODE? (you don't!) +integrator.renormalize_abundances = 0 +# tolerances +integrator.rtol_spec = 1.0e-4 +integrator.atol_spec = 1.0e-4 From ddfa289dbf715149c3b72f45d828963e0c66b701 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 30 Aug 2023 14:24:47 -0600 Subject: [PATCH 002/165] added rotation --- src/PopIII/popiii.cpp | 57 ++++++++++++++++++++++++++++--------------- tests/PopIII.in | 4 +-- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index eff4d5e79..5f42235f2 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -69,6 +69,7 @@ template <> struct SimulationData { // cloud parameters amrex::Real R_sphere{}; amrex::Real rho_sphere{}; + amrex::Real omega_sphere{}; amrex::Real small_temp; amrex::Real small_dens; @@ -248,21 +249,17 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: } } - state.T = userData_.temperature; // find the density in g/cm^3 - Real rhotot = 0.0_rt; + Real rhotot = 0.0; for (int n = 0; n < NumSpec; ++n) { state.xn[n] = numdens[n]; rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS } - state.rho = rhotot; - // normalize -- just in case - std::array mfracs = {-1.0}; - Real msum = 0.0_rt; + Real msum = 0.0; for (int n = 0; n < NumSpec; ++n) { mfracs[n] = state.xn[n] * spmasses[n] / rhotot; msum += mfracs[n]; @@ -274,15 +271,13 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } - // call the EOS to set initial internal energy e - eos(eos_input_rt, state); - amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); // cloud parameters const double R_sphere = userData_.R_sphere; + const double omega_sphere = userData_.omega_sphere; auto const &dvx = userData_.dvx.const_table(); auto const &dvy = userData_.dvy.const_table(); @@ -293,25 +288,42 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real const y = prob_lo[1] + (j + static_cast(0.5)) * dx[1]; amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx[2]; amrex::Real const r = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2) + std::pow(z - z0, 2)); + amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); - amrex::Real rho = 0; + amrex::Real phi = 0.0; + + if (x-x0 != 0.0) { + phi = atan((y-y0) / (x-x0)); + } else { + phi = M_PI / 2.0; + } + + + double vx = 0; + double vy = 0; + double vz = 0; if (r <= R_sphere) { - rho = rhotot; + state.rho = 1e-20; // rhotot; + state.T = userData_.temperature; + vx = dvx(i, j, k) - distxy * omega_sphere * std::abs(std::sin(phi)) * std::copysign(1.0, y-y0); + vy = dvx(i, j, k) + distxy * omega_sphere * std::cos(phi) * std::copysign(1.0, x-x0); + vz = dvx(i, j, k); + } else { - rho = 0.01*rhotot; + state.rho = 0.01 * 1e-20; // rhotot; + state.T = 1e2 * userData_.temperature; } - amrex::Real e = rho * state.e; + // call the EOS to set initial internal energy e + eos(eos_input_rt, state); + amrex::Real e = state.rho * state.e; - double vx = dvx(i, j, k); - double vy = dvy(i, j, k); - double vz = dvz(i, j, k); - state_cc(i, j, k, HydroSystem::density_index) = rho; - state_cc(i, j, k, HydroSystem::x1Momentum_index) = rho * vx; - state_cc(i, j, k, HydroSystem::x2Momentum_index) = rho * vy; - state_cc(i, j, k, HydroSystem::x3Momentum_index) = rho * vz; + state_cc(i, j, k, HydroSystem::density_index) = state.rho; + state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; + state_cc(i, j, k, HydroSystem::x2Momentum_index) = state.rho * vy; + state_cc(i, j, k, HydroSystem::x3Momentum_index) = state.rho * vz; state_cc(i, j, k, HydroSystem::internalEnergy_index) = e; Real const Egas = RadSystem::ComputeEgasFromEint(rho, rho*vx, rho*vy, rho*vz, e); @@ -374,6 +386,10 @@ auto problem_main() -> int Real rho_sphere{}; pp.query("cloud_density", rho_sphere); + // cloud angular velocity + Real omega_sphere{}; + pp.query("cloud_omega", omega_sphere); + // boundary conditions const int ncomp_cc = Physics_Indices::nvarTotal_cc; @@ -392,6 +408,7 @@ auto problem_main() -> int sim.userData_.R_sphere = R_sphere; sim.userData_.rho_sphere = rho_sphere; + sim.userData_.omega_sphere = omega_sphere; // initialize sim.setInitialConditions(); diff --git a/tests/PopIII.in b/tests/PopIII.in index 00b06ac89..90c74ba4e 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -33,7 +33,7 @@ plotfile_interval = 20 checkpoint_interval = 100 perturb.cloud_radius = 3.086e18 - +perturb.cloud_omega = 2.016008E-14 # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" @@ -44,7 +44,7 @@ amrex.throw_exception = 0 amrex.signal_handling = 1 primordial_chem.enabled = 1 -primordial_chem.temperature = 1e2 +primordial_chem.temperature = 265 primordial_chem.small_temp = 1.e1 primordial_chem.small_dens = 1.e-60 From bbbbe83c8d1465575ec4a5871d58afa675f54df0 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 1 Sep 2023 15:31:03 -0600 Subject: [PATCH 003/165] added minimum density below which chemistry should not be performed --- src/Chemistry.hpp | 7 ++++++- src/RadhydroSimulation.hpp | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index 1a30a0c99..6172fd3e0 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -31,7 +31,7 @@ namespace quokka::chemistry AMREX_GPU_DEVICE void chemburner(burn_t &chemstate, const Real dt); -template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed) +template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed, const Real min_density_allowed) { for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { @@ -64,6 +64,11 @@ template void computeChemistry(amrex::MultiFab &mf, const R chemstate.xn[nn] = inmfracs[nn]; } + // dont do chemistry in cells with densities below the minimum density specified + if (rho < min_density_allowed) { + return; + } + // stop the test if we have reached very high densities if (rho > max_density_allowed) { amrex::Abort("Density exceeded max_density_allowed!"); diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 0b6311229..9b0536922 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -99,6 +99,8 @@ template class RadhydroSimulation : public AMRSimulation::max(); + Real min_density_allowed = std::numeric_limits::min(); + quokka::cooling::cloudy_tables cloudyTables_; std::string coolingTableFilename_{}; @@ -340,6 +342,7 @@ template void RadhydroSimulation::readParmParse( amrex::ParmParse hpp("primordial_chem"); hpp.query("enabled", enableChemistry_); hpp.query("max_density_allowed", max_density_allowed); + hpp.query("min_density_allowed", min_density_allowed); } #endif @@ -465,7 +468,7 @@ void RadhydroSimulation::addStrangSplitSourcesWithBuiltin(amrex::Mult #ifdef PRIMORDIAL_CHEM if (enableChemistry_ == 1) { // compute chemistry - quokka::chemistry::computeChemistry(state, dt, max_density_allowed); + quokka::chemistry::computeChemistry(state, dt, max_density_allowed, min_density_allowed); } #endif From 3299e7357d4450f7c99e590c8be5773bf75d567a Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 1 Sep 2023 15:31:29 -0600 Subject: [PATCH 004/165] final touches --- src/PopIII/popiii.cpp | 86 +++++++++++++++++++++++-------------------- tests/PopIII.in | 16 ++++---- 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 5f42235f2..2bc8a6064 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -139,7 +139,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() pp.query("primary_species_13", userData_.primary_species_13); pp.query("primary_species_14", userData_.primary_species_14); - eos_init(userData_.small_temp, userData_.small_dens); + //eos_init(userData_.small_temp, userData_.small_dens); network_init(); @@ -165,10 +165,8 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() const Real rho_sph = userData_.rho_sphere; const Real rms_dv_target = 1.8e5; - const Real rms_Mach_target = 1.0; const Real rms_dv_actual = userData_.dv_rms_generated; userData_.rescale_factor = rms_dv_target / rms_dv_actual; - amrex::Print() << "rms Mach target = " << rms_Mach_target << "\n"; // copy to GPU userData_.dvx.resize(pinned_dvx.lo(), pinned_dvx.hi()); @@ -193,7 +191,6 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; - burn_t state; std::array numdens = {-1.0}; @@ -250,27 +247,6 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: } - // find the density in g/cm^3 - Real rhotot = 0.0; - for (int n = 0; n < NumSpec; ++n) { - state.xn[n] = numdens[n]; - rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS - } - - // normalize -- just in case - std::array mfracs = {-1.0}; - Real msum = 0.0; - for (int n = 0; n < NumSpec; ++n) { - mfracs[n] = state.xn[n] * spmasses[n] / rhotot; - msum += mfracs[n]; - } - - for (int n = 0; n < NumSpec; ++n) { - mfracs[n] /= msum; - // use the normalized mass fractions to obtain the corresponding number densities - state.xn[n] = mfracs[n] * rhotot / spmasses[n]; - } - amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); @@ -278,6 +254,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: // cloud parameters const double R_sphere = userData_.R_sphere; const double omega_sphere = userData_.omega_sphere; + const double renorm_amp = userData_.rescale_factor; auto const &dvx = userData_.dvx.const_table(); auto const &dvy = userData_.dvy.const_table(); @@ -290,15 +267,31 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real const r = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2) + std::pow(z - z0, 2)); amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); - amrex::Real phi = 0.0; + burn_t state; + // find the density in g/cm^3 + Real rhotot = 0.0; + for (int n = 0; n < NumSpec; ++n) { + state.xn[n] = numdens[n]; + rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS + } + + // normalize -- just in case + std::array mfracs = {-1.0}; + Real msum = 0.0; + for (int n = 0; n < NumSpec; ++n) { + mfracs[n] = state.xn[n] * spmasses[n] / rhotot; + msum += mfracs[n]; + } - if (x-x0 != 0.0) { - phi = atan((y-y0) / (x-x0)); - } else { - phi = M_PI / 2.0; + for (int n = 0; n < NumSpec; ++n) { + mfracs[n] /= msum; + // use the normalized mass fractions to obtain the corresponding number densities + state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } + amrex::Real phi = atan2((y-y0), (x-x0)); + double vx = 0; double vy = 0; double vz = 0; @@ -306,32 +299,35 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: if (r <= R_sphere) { state.rho = 1e-20; // rhotot; state.T = userData_.temperature; - vx = dvx(i, j, k) - distxy * omega_sphere * std::abs(std::sin(phi)) * std::copysign(1.0, y-y0); - vy = dvx(i, j, k) + distxy * omega_sphere * std::cos(phi) * std::copysign(1.0, x-x0); - vz = dvx(i, j, k); + vx = renorm_amp * dvx(i, j, k); + vy = renorm_amp * dvy(i, j, k); + vz = renorm_amp * dvz(i, j, k); + + // add rotation to vx and vy + vx += - distxy * omega_sphere * std::sin(phi); + vy += distxy * omega_sphere * std::cos(phi); } else { state.rho = 0.01 * 1e-20; // rhotot; - state.T = 1e2 * userData_.temperature; + state.T = 1e2 * userData_.temperature; // can use EOS.hpp function to solve for temp } // call the EOS to set initial internal energy e eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; - state_cc(i, j, k, HydroSystem::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; state_cc(i, j, k, HydroSystem::x2Momentum_index) = state.rho * vy; state_cc(i, j, k, HydroSystem::x3Momentum_index) = state.rho * vz; state_cc(i, j, k, HydroSystem::internalEnergy_index) = e; - Real const Egas = RadSystem::ComputeEgasFromEint(rho, rho*vx, rho*vy, rho*vz, e); + Real const Egas = RadSystem::ComputeEgasFromEint(state.rho, state.rho*vx, state.rho*vy, state.rho*vz, e); state_cc(i, j, k, HydroSystem::energy_index) = Egas; for (int nn = 0; nn < NumSpec; ++nn) { state_cc(i, j, k, HydroSystem::scalar0_index + nn) = - mfracs[nn] * rho; // we use partial densities and not mass fractions + mfracs[nn] * state.rho; // we use partial densities and not mass fractions } }); @@ -341,7 +337,6 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra { // refine on Jeans length const int N_cells = 32; // inverse of the 'Jeans number' [Truelove et al. (1997)] - const amrex::Real cs = quokka::EOS_Traits::cs_isothermal; const amrex::Real dx = geom[lev].CellSizeArray()[0]; const amrex::Real G = Gconst_; @@ -349,7 +344,13 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra auto tag = tags.arrays(); amrex::ParallelFor(tags, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + Real pressure = HydroSystem::ComputePressure(state[bx], i, j, k); + amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + + amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); if (l_Jeans < (N_cells * dx)) { @@ -361,14 +362,19 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const { // compute derived variables and save in 'mf' - if (dname == "log_density") { + if (dname == "temperature") { const int ncomp = ncomp_cc_in; auto const &state = state_new_cc_[lev].const_arrays(); auto output = mf.arrays(); amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + Real const rho = state[bx](i, j, k, HydroSystem::density_index); - output[bx](i, j, k, ncomp) = std::log10(rho); + amrex::Real Eint = state[bx](i, j, k, HydroSystem::internalEnergy_index); + + amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + + output[bx](i, j, k, ncomp) = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); }); } } diff --git a/tests/PopIII.in b/tests/PopIII.in index 90c74ba4e..b8f3cc074 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -13,23 +13,23 @@ amr.v = 1 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 128 128 128 +amr.n_cell = 512 512 512 amr.max_level = 0 # number of levels = max_level + 1 -amr.blocking_factor = 16 # grid size must be divisible by this -amr.max_grid_size = 64 # at least 128 for GPUs +amr.blocking_factor = 32 # grid size must be divisible by this +amr.max_grid_size = 128 # at least 128 for GPUs amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells amr.grid_eff = 0.7 # default hydro.reconstruction_order = 3 # PPM cfl = 0.2 -max_timesteps = 300 +max_timesteps = 10 stop_time = 1e16 do_reflux = 1 do_subcycle = 0 -ascent_interval = 10 -plotfile_interval = 20 +#ascent_interval = 10 +plotfile_interval = 20 #100 checkpoint_interval = 100 perturb.cloud_radius = 3.086e18 @@ -38,7 +38,7 @@ perturb.cloud_omega = 2.016008E-14 # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" -derived_vars = log_density +derived_vars = temperature amrex.throw_exception = 0 amrex.signal_handling = 1 @@ -49,6 +49,8 @@ primordial_chem.temperature = 265 primordial_chem.small_temp = 1.e1 primordial_chem.small_dens = 1.e-60 primordial_chem.max_density_allowed = 3e-6 +primordial_chem.min_density_allowed = 1e-21 + #ntot Tgas E H- D- H HE H2 HD D H+ HE+ H2+ D+ HD+ HE++ primordial_chem.primary_species_1 = 1e-4 From 1726bb62487101c33ca55ca14e39e01174d7bf81 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 1 Sep 2023 15:36:41 -0600 Subject: [PATCH 005/165] update with popiii --- src/CMakeLists.txt | 99 ++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ee2a28177..efd2d8d42 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,51 +105,54 @@ set (QuokkaObjSources "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp" "${CMAKE_CURRENT_SO #endif() #link_libraries(QuokkaObj) -add_subdirectory(StarCluster) -add_subdirectory(PrimordialChem) - -add_subdirectory(Advection) -add_subdirectory(Advection2D) -add_subdirectory(AdvectionSemiellipse) - - -add_subdirectory(HydroBlast2D) -add_subdirectory(HydroBlast3D) -add_subdirectory(HydroContact) -add_subdirectory(HydroKelvinHelmholz) -add_subdirectory(HydroLeblanc) -add_subdirectory(HydroRichtmeyerMeshkov) -add_subdirectory(HydroShocktube) -add_subdirectory(HydroShocktubeCMA) -add_subdirectory(HydroShuOsher) -add_subdirectory(HydroSMS) -add_subdirectory(HydroVacuum) -add_subdirectory(HydroWave) -add_subdirectory(HydroQuirk) -add_subdirectory(HydroHighMach) -add_subdirectory(RayleighTaylor2D) -add_subdirectory(RayleighTaylor3D) -add_subdirectory(RadBeam) -add_subdirectory(RadForce) -add_subdirectory(RadMarshak) -add_subdirectory(RadMarshakCGS) -add_subdirectory(RadMarshakAsymptotic) -add_subdirectory(RadMatterCoupling) -add_subdirectory(RadMatterCouplingRSLA) -add_subdirectory(RadPulse) -add_subdirectory(RadShadow) -add_subdirectory(RadStreaming) -add_subdirectory(RadSuOlson) -add_subdirectory(RadTophat) -add_subdirectory(RadTube) - -add_subdirectory(RadhydroShell) -add_subdirectory(RadhydroShock) -add_subdirectory(RadhydroShockCGS) - -add_subdirectory(ODEIntegration) -add_subdirectory(Cooling) -add_subdirectory(ShockCloud) -add_subdirectory(PassiveScalar) -add_subdirectory(FCQuantities) -add_subdirectory(SphericalCollapse) +add_subdirectory(PopIII) + + +##add_subdirectory(StarCluster) +##add_subdirectory(PrimordialChem) + +##add_subdirectory(Advection) +##add_subdirectory(Advection2D) +##add_subdirectory(AdvectionSemiellipse) + + +#add_subdirectory(HydroBlast2D) +#add_subdirectory(HydroBlast3D) +#add_subdirectory(HydroContact) +#add_subdirectory(HydroKelvinHelmholz) +#add_subdirectory(HydroLeblanc) +#add_subdirectory(HydroRichtmeyerMeshkov) +#add_subdirectory(HydroShocktube) +#add_subdirectory(HydroShocktubeCMA) +#add_subdirectory(HydroShuOsher) +#add_subdirectory(HydroSMS) +#add_subdirectory(HydroVacuum) +#add_subdirectory(HydroWave) +#add_subdirectory(HydroQuirk) +#add_subdirectory(HydroHighMach) +#add_subdirectory(RayleighTaylor2D) +#add_subdirectory(RayleighTaylor3D) +#add_subdirectory(RadBeam) +#add_subdirectory(RadForce) +#add_subdirectory(RadMarshak) +#add_subdirectory(RadMarshakCGS) +#add_subdirectory(RadMarshakAsymptotic) +#add_subdirectory(RadMatterCoupling) +#add_subdirectory(RadMatterCouplingRSLA) +#add_subdirectory(RadPulse) +#add_subdirectory(RadShadow) +#add_subdirectory(RadStreaming) +#add_subdirectory(RadSuOlson) +#add_subdirectory(RadTophat) +#add_subdirectory(RadTube) + +#add_subdirectory(RadhydroShell) +#add_subdirectory(RadhydroShock) +#add_subdirectory(RadhydroShockCGS) + +#add_subdirectory(ODEIntegration) +#add_subdirectory(Cooling) +#add_subdirectory(ShockCloud) +#add_subdirectory(PassiveScalar) +#add_subdirectory(FCQuantities) +#add_subdirectory(SphericalCollapse) From 6b9864202cab8afe7efb62b06132e459c380efcc Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 00:17:06 +0200 Subject: [PATCH 006/165] bug fix --- src/PopIII/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index 72ce0a0ff..4eff54330 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -13,7 +13,7 @@ if (AMReX_SPACEDIM EQUAL 3) target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM if(AMReX_GPU_BACKEND MATCHES "CUDA") - setup_target_for_cuda_compilation(star_cluster) + setup_target_for_cuda_compilation(popiii) endif() execute_process( @@ -23,9 +23,9 @@ if (AMReX_SPACEDIM EQUAL 3) ) add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) - add_test(NAME PopIII COMMAND popiii popiii.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) - set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP StarCluster_fixture) - set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED StarCluster_fixture) + add_test(NAME PopIII COMMAND popiii PopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP PopIII_fixture) + set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED PopIII_fixture) # AMR test only works on Setonix because Gadi and avatar do not have enough memory per GPU # add_test(NAME PopIIIAMR COMMAND popiii popiii_AMR.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) From 3c9ed5622d591c57188179dc9ffc407ca3636b9e Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 00:17:21 +0200 Subject: [PATCH 007/165] decrease timesteps --- tests/PopIII.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index b8f3cc074..3098c7b7c 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -13,7 +13,7 @@ amr.v = 1 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 512 512 512 +amr.n_cell = 128 128 128 amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this amr.max_grid_size = 128 # at least 128 for GPUs From b8774e650469644b59db3fd6f80d6a6e2ec2466d Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 1 Sep 2023 16:27:38 -0600 Subject: [PATCH 008/165] use numdens --- src/PopIII/popiii.cpp | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 2bc8a6064..2ce21ecb9 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -68,7 +68,7 @@ template <> struct SimulationData { // cloud parameters amrex::Real R_sphere{}; - amrex::Real rho_sphere{}; + amrex::Real numdens_init{}; amrex::Real omega_sphere{}; amrex::Real small_temp; @@ -161,10 +161,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; - const Real R_sphere = userData_.R_sphere; - const Real rho_sph = userData_.rho_sphere; - - const Real rms_dv_target = 1.8e5; + const Real rms_dv_target = 1.8050e5; const Real rms_dv_actual = userData_.dv_rms_generated; userData_.rescale_factor = rms_dv_target / rms_dv_actual; @@ -255,6 +252,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: const double R_sphere = userData_.R_sphere; const double omega_sphere = userData_.omega_sphere; const double renorm_amp = userData_.rescale_factor; + const double numdens_init = userData_.numdens; auto const &dvx = userData_.dvx.const_table(); auto const &dvy = userData_.dvy.const_table(); @@ -268,10 +266,10 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); burn_t state; - // find the density in g/cm^3 - Real rhotot = 0.0; + amrex::Real rhotot = 0.0; + for (int n = 0; n < NumSpec; ++n) { - state.xn[n] = numdens[n]; + state.xn[n] = numdens[n] * numdens_init; rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS } @@ -297,18 +295,18 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: double vz = 0; if (r <= R_sphere) { - state.rho = 1e-20; // rhotot; + state.rho = rhotot; // rhotot; state.T = userData_.temperature; vx = renorm_amp * dvx(i, j, k); vy = renorm_amp * dvy(i, j, k); vz = renorm_amp * dvz(i, j, k); // add rotation to vx and vy - vx += - distxy * omega_sphere * std::sin(phi); + vx += (-1.0) * distxy * omega_sphere * std::sin(phi); vy += distxy * omega_sphere * std::cos(phi); } else { - state.rho = 0.01 * 1e-20; // rhotot; + state.rho = 0.01 * rhotot; // rhotot; state.T = 1e2 * userData_.temperature; // can use EOS.hpp function to solve for temp } @@ -389,8 +387,8 @@ auto problem_main() -> int pp.query("cloud_radius", R_sphere); // cloud density - Real rho_sphere{}; - pp.query("cloud_density", rho_sphere); + Real numdens_init{}; + pp.query("cloud_numdens", numdens_init); // cloud angular velocity Real omega_sphere{}; @@ -413,7 +411,7 @@ auto problem_main() -> int sim.densityFloor_ = 0.01; sim.userData_.R_sphere = R_sphere; - sim.userData_.rho_sphere = rho_sphere; + sim.userData_.numdens_init = numdens_init; sim.userData_.omega_sphere = omega_sphere; // initialize From b1ef513727efc00f20c1cafbb7f6cb867c261974 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 1 Sep 2023 16:57:26 -0600 Subject: [PATCH 009/165] verbose --- src/PopIII/popiii.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 2ce21ecb9..35ac0bf39 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -314,6 +314,8 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; + amrex::Print() << "cell " << i << j << k << state.rho << vx << vy << vz << e << state.T <::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; state_cc(i, j, k, HydroSystem::x2Momentum_index) = state.rho * vy; From 44272c251f2609fc96da4d5ea9aa564d0ef00edb Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 00:58:58 +0200 Subject: [PATCH 010/165] fix bug --- src/PopIII/popiii.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 2ce21ecb9..df16ce402 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -252,7 +252,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: const double R_sphere = userData_.R_sphere; const double omega_sphere = userData_.omega_sphere; const double renorm_amp = userData_.rescale_factor; - const double numdens_init = userData_.numdens; + const double numdens_init = userData_.numdens_init; auto const &dvx = userData_.dvx.const_table(); auto const &dvy = userData_.dvy.const_table(); From 65ece5079a8276d773767e4c4dfd7df6afa51b69 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 00:59:34 +0200 Subject: [PATCH 011/165] define proper values --- tests/PopIII.in | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index 3098c7b7c..29ef467db 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -34,6 +34,8 @@ checkpoint_interval = 100 perturb.cloud_radius = 3.086e18 perturb.cloud_omega = 2.016008E-14 +perturb.numdens_init = 0.90861183E+004 + # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" @@ -44,29 +46,28 @@ amrex.throw_exception = 0 amrex.signal_handling = 1 primordial_chem.enabled = 1 -primordial_chem.temperature = 265 +primordial_chem.temperature = 0.26415744E+003 primordial_chem.small_temp = 1.e1 primordial_chem.small_dens = 1.e-60 primordial_chem.max_density_allowed = 3e-6 primordial_chem.min_density_allowed = 1e-21 - - -#ntot Tgas E H- D- H HE H2 HD D H+ HE+ H2+ D+ HD+ HE++ -primordial_chem.primary_species_1 = 1e-4 -primordial_chem.primary_species_2 = 1e-4 -primordial_chem.primary_species_3 = 1e0 -primordial_chem.primary_species_4 = 1e-40 -primordial_chem.primary_species_5 = 1e-40 -primordial_chem.primary_species_6 = 1e-40 -primordial_chem.primary_species_7 = 1e-40 -primordial_chem.primary_species_8 = 1e-40 -primordial_chem.primary_species_9 = 1e-6 -primordial_chem.primary_species_10 = 1e-40 -primordial_chem.primary_species_11 = 1e-40 -primordial_chem.primary_species_12 = 1e-40 -primordial_chem.primary_species_13 = 1e-40 -primordial_chem.primary_species_14 = 0.0775e0 +#format in krome fort22_wD.dat file: E H- D- H HE H2 HD D H+ HE+ H2+ D+ HD+ HE++ +#format in quokka: E H+ H H- D+ D H2+ D- H2 HD+ HD HE++ HE+ HE +primordial_chem.primary_species_1 = 0.88499253E-006 +primordial_chem.primary_species_2 = 0.88498062E-006 +primordial_chem.primary_species_3 = 0.99932238E+000 +primordial_chem.primary_species_4 = 0.54719550E-013 +primordial_chem.primary_species_5 = 0.21957612E-010 +primordial_chem.primary_species_6 = 0.29920413E-004 +primordial_chem.primary_species_7 = 0.58304958E-015 +primordial_chem.primary_species_8 = 0.22122496E-017 +primordial_chem.primary_species_9 = 0.38932607E-003 +primordial_chem.primary_species_10 = 0.36774691E-019 +primordial_chem.primary_species_11 = 0.79574711E-007 +primordial_chem.primary_species_12 = 0.39651766E-050 +primordial_chem.primary_species_13 = 0.24136647E-043 +primordial_chem.primary_species_14 = 0.77500001E-001 # integrator runtime parameters From 6e22e8b9b4443ac5f95fff941a5d5f8054f0f6a1 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 01:08:06 +0200 Subject: [PATCH 012/165] fix bug --- src/PopIII/popiii.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 2b6bd26ce..b8455b220 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -139,7 +139,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() pp.query("primary_species_13", userData_.primary_species_13); pp.query("primary_species_14", userData_.primary_species_14); - //eos_init(userData_.small_temp, userData_.small_dens); + eos_init(userData_.small_temp, userData_.small_dens); network_init(); @@ -314,7 +314,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; - amrex::Print() << "cell " << i << j << k << state.rho << vx << vy << vz << e << state.T <::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; From a41c1a4b946bc925f14a66acd1b39f45952b2b40 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 1 Sep 2023 17:14:25 -0600 Subject: [PATCH 013/165] try --- src/PopIII/popiii.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index b8455b220..9ff7af6e6 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -287,6 +287,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } + amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; amrex::Real phi = atan2((y-y0), (x-x0)); @@ -314,7 +315,6 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; - amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << vx << " " << vy << " " << vz << " " << e << " " << state.T << std::endl; state_cc(i, j, k, HydroSystem::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; From 25cc0278c4a5918855c98c8a6c14b1baeb42f322 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 01:18:36 +0200 Subject: [PATCH 014/165] fixed bug --- src/PopIII/popiii.cpp | 2 +- tests/PopIII.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 9ff7af6e6..26b17342e 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -287,7 +287,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } - amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; + //amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; amrex::Real phi = atan2((y-y0), (x-x0)); diff --git a/tests/PopIII.in b/tests/PopIII.in index 29ef467db..8c8f25aa4 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -34,7 +34,7 @@ checkpoint_interval = 100 perturb.cloud_radius = 3.086e18 perturb.cloud_omega = 2.016008E-14 -perturb.numdens_init = 0.90861183E+004 +perturb.cloud_numdens = 0.90861183E+004 # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/ From 6814273bf5bfab2bd10fb3ab280e2f5cfb759cbe Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 05:39:43 +0200 Subject: [PATCH 015/165] try2 --- src/PopIII/popiii.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 26b17342e..aff833dc8 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -287,7 +287,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } - //amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; + amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; amrex::Real phi = atan2((y-y0), (x-x0)); @@ -315,6 +315,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; + amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; state_cc(i, j, k, HydroSystem::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; From 4fc5ffa96ffd1187ea46a8e35e76e76e90bf0873 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 2 Sep 2023 06:14:36 +0200 Subject: [PATCH 016/165] reset density floor --- src/PopIII/popiii.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index aff833dc8..abf7d7c19 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -287,7 +287,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } - amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; + //amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; amrex::Real phi = atan2((y-y0), (x-x0)); @@ -315,7 +315,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; - amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; + //amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; state_cc(i, j, k, HydroSystem::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; @@ -411,7 +411,7 @@ auto problem_main() -> int // Problem initialization RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity - sim.densityFloor_ = 0.01; + sim.densityFloor_ = 1e-60; sim.userData_.R_sphere = R_sphere; sim.userData_.numdens_init = numdens_init; From f17b2714ff9b19e89ee11b4cec332eac3e86ec39 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 02:51:17 +0200 Subject: [PATCH 017/165] define temp outside parallelFor --- src/PopIII/popiii.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index abf7d7c19..980ea37a8 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -253,6 +253,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: const double omega_sphere = userData_.omega_sphere; const double renorm_amp = userData_.rescale_factor; const double numdens_init = userData_.numdens_init; + const double core_temp = userData_.temperature; auto const &dvx = userData_.dvx.const_table(); auto const &dvy = userData_.dvy.const_table(); @@ -296,8 +297,8 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: double vz = 0; if (r <= R_sphere) { - state.rho = rhotot; // rhotot; - state.T = userData_.temperature; + state.rho = rhotot; + state.T = core_temp; vx = renorm_amp * dvx(i, j, k); vy = renorm_amp * dvy(i, j, k); vz = renorm_amp * dvz(i, j, k); @@ -307,8 +308,8 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: vy += distxy * omega_sphere * std::cos(phi); } else { - state.rho = 0.01 * rhotot; // rhotot; - state.T = 1e2 * userData_.temperature; // can use EOS.hpp function to solve for temp + state.rho = 0.01 * rhotot; + state.T = 1e2 * core_temp; // can use EOS.hpp function to solve for temp } // call the EOS to set initial internal energy e From 60fba77ce50a7db9f9a3dce846b63e7192243acb Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 05:50:23 +0200 Subject: [PATCH 018/165] use BL_Profile --- src/Chemistry.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index 6172fd3e0..73582cba0 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -34,6 +34,7 @@ AMREX_GPU_DEVICE void chemburner(burn_t &chemstate, const Real dt); template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed, const Real min_density_allowed) { + BL_PROFILE("Chemistry::computeChemistry()"); for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); auto const &state = mf.array(iter); From 5dbb67cb520b0d252f4bb491d1978720e0e74bd7 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 19:36:46 +0200 Subject: [PATCH 019/165] inputs for perturbations.py direcrly read from the .in file --- src/PopIII/CMakeLists.txt | 49 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index 4eff54330..b6d3c6a6c 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -11,18 +11,63 @@ if (AMReX_SPACEDIM EQUAL 3) add_executable(popiii popiii.cpp TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM - + if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(popiii) endif() + #need h5py to run perturbations.py file below execute_process( COMMAND Python3::Interpreter -c "h5py" RESULT_VARIABLE EXIT_CODE OUTPUT_QUIET ) - add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + #re-run cmake if this file is changed + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/PopIII.in) + + # Read the 'PopIII.in' file line by line + file(READ ${CMAKE_SOURCE_DIR}/tests/PopIII.in pop_in_contents) + + # Split the file contents into lines + string(REPLACE "\n" ";" pop_in_lines "${pop_in_contents}") + + # Iterate through the lines and look for lines starting with 'amr.n_cell' + foreach(line IN LISTS pop_in_lines) + if (line MATCHES "^amr\\.n_cell.*") + set(match_line "${line}") + break() + endif() + endforeach() + + if (match_line) + message(STATUS "picked line is ${match_line}") + # Use a regular expression to find consecutive digit + string(REGEX MATCHALL "[0-9]+" digits_list "${match_line}") + # Iterate through the matched digits and extract the first one + list(GET digits_list 0 first_digits) + # Check if matched digits were found + if (first_digits) + #first_digits give the box size, but you also want box size / 2 as kmax in the script below + math(EXPR int_first_digits "${first_digits}") + # Check if the conversion was successful + if (int_first_digits) + # Divide the integer by two + math(EXPR result "${int_first_digits} / 2") + else() + message(FATAL_ERROR "Conversion to integer failed, so cannot find kmax!") + endif() + message(STATUS "Will create initial perturbations of box size ${first_digits} and kmax ${result}") + + else() + message(FATAL_ERROR "No box size found to create initial perturbations for!") + endif() + else() + message(FATAL_ERROR "No matching line found in ${CMAKE_SOURCE_DIR}/tests/PopIII.in!") + endif() + + + add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) add_test(NAME PopIII COMMAND popiii PopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP PopIII_fixture) set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED PopIII_fixture) From 0e90457fd088354bdfbc24dce724c82b39aec39d Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 20:39:32 +0200 Subject: [PATCH 020/165] updated cmakelists --- src/CMakeLists.txt | 96 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d3a27cee..dd41d327c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,51 +109,51 @@ set (QuokkaObjSources "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp" "${CMAKE_CURRENT_SO add_subdirectory(PopIII) -##add_subdirectory(StarCluster) -##add_subdirectory(PrimordialChem) - -##add_subdirectory(Advection) -##add_subdirectory(Advection2D) -##add_subdirectory(AdvectionSemiellipse) - - -#add_subdirectory(HydroBlast2D) -#add_subdirectory(HydroBlast3D) -#add_subdirectory(HydroContact) -#add_subdirectory(HydroKelvinHelmholz) -#add_subdirectory(HydroLeblanc) -#add_subdirectory(HydroRichtmeyerMeshkov) -#add_subdirectory(HydroShocktube) -#add_subdirectory(HydroShocktubeCMA) -#add_subdirectory(HydroShuOsher) -#add_subdirectory(HydroSMS) -#add_subdirectory(HydroVacuum) -#add_subdirectory(HydroWave) -#add_subdirectory(HydroQuirk) -#add_subdirectory(HydroHighMach) -#add_subdirectory(RayleighTaylor2D) -#add_subdirectory(RayleighTaylor3D) -#add_subdirectory(RadBeam) -#add_subdirectory(RadForce) -#add_subdirectory(RadMarshak) -#add_subdirectory(RadMarshakCGS) -#add_subdirectory(RadMarshakAsymptotic) -#add_subdirectory(RadMatterCoupling) -#add_subdirectory(RadMatterCouplingRSLA) -#add_subdirectory(RadPulse) -#add_subdirectory(RadShadow) -#add_subdirectory(RadStreaming) -#add_subdirectory(RadSuOlson) -#add_subdirectory(RadTophat) -#add_subdirectory(RadTube) - -#add_subdirectory(RadhydroShell) -#add_subdirectory(RadhydroShock) -#add_subdirectory(RadhydroShockCGS) - -#add_subdirectory(ODEIntegration) -#add_subdirectory(Cooling) -#add_subdirectory(ShockCloud) -#add_subdirectory(PassiveScalar) -#add_subdirectory(FCQuantities) -#add_subdirectory(SphericalCollapse) +add_subdirectory(StarCluster) +add_subdirectory(PrimordialChem) + +add_subdirectory(Advection) +add_subdirectory(Advection2D) +add_subdirectory(AdvectionSemiellipse) + + +add_subdirectory(HydroBlast2D) +add_subdirectory(HydroBlast3D) +add_subdirectory(HydroContact) +add_subdirectory(HydroKelvinHelmholz) +add_subdirectory(HydroLeblanc) +add_subdirectory(HydroRichtmeyerMeshkov) +add_subdirectory(HydroShocktube) +add_subdirectory(HydroShocktubeCMA) +add_subdirectory(HydroShuOsher) +add_subdirectory(HydroSMS) +add_subdirectory(HydroVacuum) +add_subdirectory(HydroWave) +add_subdirectory(HydroQuirk) +add_subdirectory(HydroHighMach) +add_subdirectory(RayleighTaylor2D) +add_subdirectory(RayleighTaylor3D) +add_subdirectory(RadBeam) +add_subdirectory(RadForce) +add_subdirectory(RadMarshak) +add_subdirectory(RadMarshakCGS) +add_subdirectory(RadMarshakAsymptotic) +add_subdirectory(RadMatterCoupling) +add_subdirectory(RadMatterCouplingRSLA) +add_subdirectory(RadPulse) +add_subdirectory(RadShadow) +add_subdirectory(RadStreaming) +add_subdirectory(RadSuOlson) +add_subdirectory(RadTophat) +add_subdirectory(RadTube) + +add_subdirectory(RadhydroShell) +add_subdirectory(RadhydroShock) +add_subdirectory(RadhydroShockCGS) + +add_subdirectory(ODEIntegration) +add_subdirectory(Cooling) +add_subdirectory(ShockCloud) +add_subdirectory(PassiveScalar) +add_subdirectory(FCQuantities) +add_subdirectory(SphericalCollapse) From a57d9c89cf94b4c1d778842b53e725bea0cff27f Mon Sep 17 00:00:00 2001 From: Piyush Sharda <34922596+psharda@users.noreply.github.com> Date: Fri, 8 Sep 2023 12:51:18 -0600 Subject: [PATCH 021/165] correct includes order Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/PopIII/popiii.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 980ea37a8..f461002b7 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -25,8 +25,8 @@ #include "RadhydroSimulation.hpp" #include "SimulationData.hpp" +#include "TurbDataReader.hpp" #include "hydro_system.hpp" -#include "radiation_system.hpp" #include "popiii.hpp" #include "TurbDataReader.hpp" From 206b7f107cbe881429deb5778104513b226279d0 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 20:53:25 +0200 Subject: [PATCH 022/165] add {} to variable declaration --- src/PopIII/popiii.cpp | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index f461002b7..69712dd50 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -71,23 +71,23 @@ template <> struct SimulationData { amrex::Real numdens_init{}; amrex::Real omega_sphere{}; - amrex::Real small_temp; - amrex::Real small_dens; - amrex::Real temperature; - amrex::Real primary_species_1; - amrex::Real primary_species_2; - amrex::Real primary_species_3; - amrex::Real primary_species_4; - amrex::Real primary_species_5; - amrex::Real primary_species_6; - amrex::Real primary_species_7; - amrex::Real primary_species_8; - amrex::Real primary_species_9; - amrex::Real primary_species_10; - amrex::Real primary_species_11; - amrex::Real primary_species_12; - amrex::Real primary_species_13; - amrex::Real primary_species_14; + amrex::Real small_temp{}; + amrex::Real small_dens{}; + amrex::Real temperature{}; + amrex::Real primary_species_1{}; + amrex::Real primary_species_2{}; + amrex::Real primary_species_3{}; + amrex::Real primary_species_4{}; + amrex::Real primary_species_5{}; + amrex::Real primary_species_6{}; + amrex::Real primary_species_7{}; + amrex::Real primary_species_8{}; + amrex::Real primary_species_9{}; + amrex::Real primary_species_10{}; + amrex::Real primary_species_11{}; + amrex::Real primary_species_12{}; + amrex::Real primary_species_13{}; + amrex::Real primary_species_14{}; }; @@ -344,7 +344,7 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra auto const &state = state_new_cc_[lev].const_arrays(); auto tag = tags.arrays(); - + amrex::Print() << "start error est " << std::endl; amrex::ParallelFor(tags, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { Real const rho = state[bx](i, j, k, HydroSystem::density_index); @@ -357,8 +357,10 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra if (l_Jeans < (N_cells * dx)) { tag[bx](i, j, k) = amrex::TagBox::SET; + printf("tagging cell for refinement \n"); } }); + amrex::Print() << "end error est " << std::endl; } template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const From b4246fbfca7fa5f255b7d9f188cc010ef28cf7f4 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 20:56:41 +0200 Subject: [PATCH 023/165] added author name --- src/PopIII/popiii.cpp | 1 + src/PopIII/popiii.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 69712dd50..49d700a25 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -5,6 +5,7 @@ //============================================================================== /// \file popiii.cpp /// \brief Defines a test problem for Pop III star formation. +/// Author: Piyush Sharda (Leiden University, 2023) /// #include #include diff --git a/src/PopIII/popiii.hpp b/src/PopIII/popiii.hpp index 6fb4cd7ce..9d3695bf3 100644 --- a/src/PopIII/popiii.hpp +++ b/src/PopIII/popiii.hpp @@ -7,6 +7,7 @@ //============================================================================== /// \file star_cluster.hpp /// \brief Defines a test problem for Pop III star formation. +/// Author: Piyush Sharda (Leiden University, 2023) /// // external headers From 3b42508320fead968868b4c20e04b6b3e51fe6c2 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:06:48 +0200 Subject: [PATCH 024/165] remove duplicate include --- src/PopIII/popiii.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 49d700a25..7d7cd0e01 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -29,7 +29,6 @@ #include "TurbDataReader.hpp" #include "hydro_system.hpp" #include "popiii.hpp" -#include "TurbDataReader.hpp" #include "actual_eos_data.H" #include "burn_type.H" From c8835a8ef1b8e4b7be7fc4d95e97614e376af3a2 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:36:18 +0200 Subject: [PATCH 025/165] include NSCBC --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dd41d327c..669756a47 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -116,7 +116,6 @@ add_subdirectory(Advection) add_subdirectory(Advection2D) add_subdirectory(AdvectionSemiellipse) - add_subdirectory(HydroBlast2D) add_subdirectory(HydroBlast3D) add_subdirectory(HydroContact) @@ -157,3 +156,4 @@ add_subdirectory(ShockCloud) add_subdirectory(PassiveScalar) add_subdirectory(FCQuantities) add_subdirectory(SphericalCollapse) +add_subdirectory(NSCBC) From 31e40327305e15437119449e7b1a62c589df26ba Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:37:37 +0200 Subject: [PATCH 026/165] added comments --- src/RadhydroSimulation.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 14cc1830c..a96f42da3 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -361,8 +361,8 @@ template void RadhydroSimulation::readParmParse( { amrex::ParmParse hpp("primordial_chem"); hpp.query("enabled", enableChemistry_); - hpp.query("max_density_allowed", max_density_allowed); - hpp.query("min_density_allowed", min_density_allowed); + hpp.query("max_density_allowed", max_density_allowed); // chemistry is not accurate for densities > 3e-6 + hpp.query("min_density_allowed", min_density_allowed); // don't do chemistry in cells with densities below the minimum density specified } #endif From d4646a09b5339ebb0cc2ead044e127046929e879 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:40:25 +0200 Subject: [PATCH 027/165] set reconstruct_eint to true --- src/PopIII/popiii.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 7d7cd0e01..4274b67be 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -42,13 +42,12 @@ struct PopIII { }; template <> struct HydroSystem_Traits { - static constexpr bool reconstruct_eint = false; + static constexpr bool reconstruct_eint = true; }; template <> struct Physics_Traits { // cell-centred static constexpr bool is_hydro_enabled = true; - static constexpr bool is_chemistry_enabled = false; // in the future, this could point to microphysics, and set to true static constexpr int numMassScalars = NumSpec; // number of chemical species static constexpr int numPassiveScalars = numMassScalars + 0; // we only have mass scalars static constexpr bool is_radiation_enabled = false; From 0a7c1076bf856bd2cc1f91095ee0e1fc2ae2e420 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:47:28 +0200 Subject: [PATCH 028/165] move perturbations to src/ --- src/StarCluster/perturbation.py | 311 -------------------------------- 1 file changed, 311 deletions(-) delete mode 100644 src/StarCluster/perturbation.py diff --git a/src/StarCluster/perturbation.py b/src/StarCluster/perturbation.py deleted file mode 100644 index d6d53e640..000000000 --- a/src/StarCluster/perturbation.py +++ /dev/null @@ -1,311 +0,0 @@ -import h5py -import numpy as np -import numpy.fft -from math import * -from optparse import OptionParser -import sys - - -def init_perturbations(n, kmin, kmax, dtype): - kx = np.zeros(n, dtype=dtype) - ky = np.zeros(n, dtype=dtype) - kz = np.zeros(n, dtype=dtype) - # perform fft k-ordering convention shifts - for j in range(0,n[1]): - for k in range(0,n[2]): - kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) - for i in range(0,n[0]): - for k in range(0,n[2]): - ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) - for i in range(0,n[0]): - for j in range(0,n[1]): - kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) - - kx = np.array(kx, dtype=dtype) - ky = np.array(ky, dtype=dtype) - kz = np.array(kz, dtype=dtype) - k = np.sqrt(np.array(kx**2+ky**2+kz**2, dtype=dtype)) - - # only use the positive frequencies - inds = np.where(np.logical_and(k**2 >= kmin**2, k**2 < (kmax+1)**2)) - nr = len(inds[0]) - - phasex = np.zeros(n, dtype=dtype) - phasex[inds] = 2.*pi*np.random.uniform(size=nr) - fx = np.zeros(n, dtype=dtype) - fx[inds] = np.random.normal(size=nr) - - phasey = np.zeros(n, dtype=dtype) - phasey[inds] = 2.*pi*np.random.uniform(size=nr) - fy = np.zeros(n, dtype=dtype) - fy[inds] = np.random.normal(size=nr) - - phasez = np.zeros(n, dtype=dtype) - phasez[inds] = 2.*pi*np.random.uniform(size=nr) - fz = np.zeros(n, dtype=dtype) - fz[inds] = np.random.normal(size=nr) - - # rescale perturbation amplitude so that low number statistics - # at low k do not throw off the desired power law scaling. - for i in range(kmin, kmax+1): - slice_inds = np.where(np.logical_and(k >= i, k < i+1)) - rescale = sqrt(np.sum(np.abs(fx[slice_inds])**2 + np.abs(fy[slice_inds])**2 + np.abs(fz[slice_inds])**2)) - fx[slice_inds] = fx[slice_inds]/rescale - fy[slice_inds] = fy[slice_inds]/rescale - fz[slice_inds] = fz[slice_inds]/rescale - - # set the power law behavior - # wave number bins - fx[inds] = fx[inds]*k[inds]**-(0.5*alpha) - fy[inds] = fy[inds]*k[inds]**-(0.5*alpha) - fz[inds] = fz[inds]*k[inds]**-(0.5*alpha) - - # add in phases - fx = np.cos(phasex)*fx + 1j*np.sin(phasex)*fx - fy = np.cos(phasey)*fy + 1j*np.sin(phasey)*fy - fz = np.cos(phasez)*fz + 1j*np.sin(phasez)*fz - - return fx, fy, fz, kx, ky, kz - - -def normalize(fx, fy, fz): - norm = np.sqrt(np.sum(fx**2 + fy**2 + fz**2)/np.product(n)) - fx = fx/norm - fy = fy/norm - fz = fz/norm - return fx, fy, fz - - -def make_perturbations(n, kmin, kmax, f_solenoidal): - fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) - if f_solenoidal != None: - k2 = kx**2+ky**2+kz**2 - # solenoidal part - fxs = 0.; fys =0.; fzs = 0. - if f_solenoidal != 0.0: - fxs = np.real(fx - kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fys = np.real(fy - ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fzs = np.real(fz - kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - ind = np.where(k2 == 0) - fxs[ind] = 0.; fys[ind] = 0.; fzs[ind] = 0. - # need to normalize this before applying relative weighting of solenoidal / compressive components - norm = np.sqrt(np.sum(fxs**2+fys**2+fzs**2)) - fxs = fxs/norm - fys = fys/norm - fzs = fzs/norm - # compressive part - # get a different random cube for the compressive part - # so that we can target the RMS solenoidal fraction, - # instead of setting a constant solenoidal fraction everywhere. - fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) - fxc = 0.; fyc =0.; fzc = 0. - if f_solenoidal != 1.0: - fxc = np.real(kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fyc = np.real(ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fzc = np.real(kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - ind = np.where(k2 == 0) - fxc[ind] = 0.; fyc[ind] = 0.; fzc[ind] = 0. - # need to normalize this before applying relative weighting of solenoidal / compressive components - norm = np.sqrt(np.sum(fxc**2+fyc**2+fzc**2)) - fxc = fxc/norm - fyc = fyc/norm - fzc = fzc/norm - # back to real space - pertx = np.real(np.fft.ifftn(f_solenoidal*fxs + (1.-f_solenoidal)*fxc)) - perty = np.real(np.fft.ifftn(f_solenoidal*fys + (1.-f_solenoidal)*fyc)) - pertz = np.real(np.fft.ifftn(f_solenoidal*fzs + (1.-f_solenoidal)*fzc)) - else: - # just convert to real space - pertx = np.real(np.fft.ifftn(fx)) - perty = np.real(np.fft.ifftn(fy)) - pertz = np.real(np.fft.ifftn(fz)) - - # subtract off COM (assuming uniform density) - pertx = pertx-np.average(pertx) - perty = perty-np.average(perty) - pertz = pertz-np.average(pertz) - # scale RMS of perturbation cube to unity - pertx, perty, pertz = normalize(pertx, perty, pertz) - return pertx, perty, pertz - - -def cut_sphere(pertx, perty, pertz, rad): - # Make radial array - x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] - x = x - (n[0]-1)/2. - y = y - (n[1]-1)/2. - z = z - (n[2]-1)/2. - r2 = x**2+y**2+z**2 - # Get range of indices we want to set to zero, and those we want to keep - idx0 = r2 > (rad*n[0]/2.0)**2 - idx1 = np.logical_not(idx0) - # Zero outside the desired region - pertx[idx0] = 0.0 - perty[idx0] = 0.0 - pertz[idx0] = 0.0 - # Recompute COM velocity, and renormalize - pertx[idx1] = pertx[idx1]-np.average(pertx[idx1]) - perty[idx1] = perty[idx1]-np.average(perty[idx1]) - pertz[idx1] = pertz[idx1]-np.average(pertz[idx1]) - pertx, perty, pertz = normalize(pertx, perty, pertz) - return pertx, perty, pertz - - -def get_erot_ke_ratio(pertx, perty, pertz, rad=-1.0): - x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] - x = x - (n[0]-1)/2. - y = y - (n[1]-1)/2. - z = z - (n[2]-1)/2. - r2 = x**2+y**2+z**2 - if rad > 0: - idx0 = r2 > (rad*n[0]/2.0)**2 - r2[idx0] = 0.0 - erot_ke_ratio = (np.sum(y*pertz-z*perty)**2 + - np.sum(z*pertx-x*pertz)**2 + - np.sum(x*perty-y*pertx)**2)/(np.sum(r2)*np.product(n)) - return erot_ke_ratio - - -def plot_spectrum1D(pertx, perty, pertz): - # plot the 1D power to check the scaling. - fx = np.abs(np.fft.fftn(pertx)) - fy = np.abs(np.fft.fftn(perty)) - fz = np.abs(np.fft.fftn(pertz)) - fx = np.abs(fx) - fy = np.abs(fy) - fz = np.abs(fz) - kx = np.zeros(n, dtype=dtype) - ky = np.zeros(n, dtype=dtype) - kz = np.zeros(n, dtype=dtype) - # perform fft k-ordering convention shifts - for j in range(0,n[1]): - for k in range(0,n[2]): - kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) - for i in range(0,n[0]): - for k in range(0,n[2]): - ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) - for i in range(0,n[0]): - for j in range(0,n[1]): - kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) - k = np.sqrt(np.array(kx**2+ky**2+kz**2,dtype=dtype)) - k1d = [] - power = [] - for i in range(kmin,kmax+1): - slice_inds = np.where(np.logical_and(k >= i, k < i+1)) - k1d.append(i+0.5) - power.append(np.sum(fx[slice_inds]**2 + fy[slice_inds]**2 + fz[slice_inds]**2)) - print(i,power[-1]) - import matplotlib.pyplot as plt - plt.loglog(k1d, power) - plt.show() - - -################### -# input parameters, read from command line -################### -parser = OptionParser() -parser.add_option('--kmin', dest='kmin', - help='minimum wavenumber.', - default=-1) -parser.add_option('--kmax', dest='kmax', - help='maximum wavenumber.', - default=-1) -parser.add_option('--size', dest='size', - help='size of each direction of data cube. default=256', - default=256) -parser.add_option('--alpha', dest='alpha', - help='negative of power law slope. (Power ~ k^-alpha) '+ - 'supersonic turbulence is near alpha=2. '+ - 'driving over a narrow band of two modes is often done with alpha=0', - default = None) -parser.add_option('--seed', dest='seed', - help='seed for random # generation. default=0', - default = 0) -parser.add_option('--f_solenoidal', dest='f_solenoidal', - help='volume RMS fraction of solenoidal component of the perturbations relative to the total. ' + - 'If --f_solenoidal=None, the motions are purely random. For low wave numbers ' + - 'the relative imporance of solenoidal to compressive may be sensitive to the ' + - 'choice of radom seed. It has been suggested (Federrath 2008) that ' + - 'f_solenoidal=2/3 is the most natural driving mode and this is currently' + - 'the suggested best-practice.', - default = -1) -parser.add_option('--sphererad', dest='rad', - help='if set, perturbations are set to zero outside spherical region, '+ - 'and the perturbation field is shifted and renormalized to keep the '+ - 'center of mass velocity at zero and the variance at unity; the '+ - 'spherical region cut out is centered at the center of the perturbation '+ - 'cube, and has a radius given by the value of this parameter, with sphererad = '+ - '1 corresponding to the spherical region going all the way to the edge of the '+ - 'perturbation cube', - default = -1.0) - -(options, args) = parser.parse_args() - -# size of the data domain -n = [int(options.size), int(options.size), int(options.size)] -# range of perturbation length scale in units of the smallest side of the domain -kmin = int(options.kmin) -kmax = int(options.kmax) -print(kmin, kmax) -if kmin > kmax or kmin < 0 or kmax < 0: - print("kmin must be < kmax, with kmin > 0, kmax > 0. See --help.") - sys.exit(0) -if kmax > floor(np.min(n))/2: - print("kmax must be <= floor(size/2). See --help.") - sys.exit(0) -f_solenoidal = options.f_solenoidal -if f_solenoidal == "None" or f_solenoidal == "none": - f_solenoidal = None -else: - f_solenoidal = float(options.f_solenoidal) - if f_solenoidal > 1. or f_solenoidal < 0.: - print("You must choose f_solenoidal. See --help.") - sys.exit(0) -alpha = options.alpha -if alpha==None: - print("You must choose a power law slope, alpha. See --help.") - sys.exit(0) -alpha = float(options.alpha) -if alpha < 0.: - print("alpha is less than zero. That's probably not what you want. See --help.") - sys.exit(0) -seed = int(options.seed) -# data precision -dtype = np.float64 -# ratio of solenoidal to compressive components -if options.f_solenoidal=="None" or options.f_solenoidal==None: - f_solenoidal = None -else: - f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) -rad = float(options.rad) -if rad > 1: - raise ValueError('sphererad is '+options.rad+', must be from 0 to 1') - -################### -# begin computation -################### - -np.random.seed(seed=seed) -pertx, perty, pertz = make_perturbations(n, kmin, kmax, f_solenoidal) -if rad > 0: - pertx, perty, pertz = cut_sphere(pertx, perty, pertz, rad) -erot_ke_ratio = get_erot_ke_ratio(pertx, perty, pertz, rad) -print("erot_ke_ratio = ", erot_ke_ratio) - -# hdf5 output -f = h5py.File('zdrv.hdf5', 'w') -ds = f['/'].create_dataset('pertx', n, dtype=np.float64) -ds[:] = pertx -ds = f['/'].create_dataset('perty', n, dtype=np.float64) -ds[:] = perty -ds = f['/'].create_dataset('pertz', n, dtype=np.float64) -ds[:] = pertz -f['/'].attrs['kmin'] = kmin -f['/'].attrs['kmax'] = kmax -f['/'].attrs['alpha'] = alpha -if f_solenoidal!=None: f['/'].attrs['f_solenoidal'] = f_solenoidal -if rad > 0: f['/'].attrs['sphererad'] = rad -f['/'].attrs['erot_ke_ratio'] = erot_ke_ratio -f['/'].attrs['seed'] = seed -f.close() - From 7a814aff498ac691949311c6e930937425a453b9 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:47:50 +0200 Subject: [PATCH 029/165] moved perturbatinos to src/ --- src/StarCluster/CMakeLists.txt | 4 +- src/perturbation.py | 311 +++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+), 2 deletions(-) create mode 100644 src/perturbation.py diff --git a/src/StarCluster/CMakeLists.txt b/src/StarCluster/CMakeLists.txt index 277525261..796f1f1c8 100644 --- a/src/StarCluster/CMakeLists.txt +++ b/src/StarCluster/CMakeLists.txt @@ -1,7 +1,7 @@ if (AMReX_SPACEDIM EQUAL 3) # Define a custom target that runs the Python script to produce the input perturbations file - add_executable(star_cluster star_cluster.cpp TurbDataReader.cpp ${QuokkaObjSources}) + add_executable(star_cluster star_cluster.cpp ../TurbDataReader.cpp ${QuokkaObjSources}) if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(star_cluster) endif() @@ -12,7 +12,7 @@ if (AMReX_SPACEDIM EQUAL 3) OUTPUT_QUIET ) - add_test(NAME ComputeStarClusterPerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + add_test(NAME ComputeStarClusterPerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) add_test(NAME StarCluster COMMAND star_cluster StarCluster.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) set_tests_properties(ComputeStarClusterPerturbations PROPERTIES FIXTURES_SETUP StarCluster_fixture) set_tests_properties(StarCluster PROPERTIES FIXTURES_REQUIRED StarCluster_fixture) diff --git a/src/perturbation.py b/src/perturbation.py new file mode 100644 index 000000000..d6d53e640 --- /dev/null +++ b/src/perturbation.py @@ -0,0 +1,311 @@ +import h5py +import numpy as np +import numpy.fft +from math import * +from optparse import OptionParser +import sys + + +def init_perturbations(n, kmin, kmax, dtype): + kx = np.zeros(n, dtype=dtype) + ky = np.zeros(n, dtype=dtype) + kz = np.zeros(n, dtype=dtype) + # perform fft k-ordering convention shifts + for j in range(0,n[1]): + for k in range(0,n[2]): + kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) + for i in range(0,n[0]): + for k in range(0,n[2]): + ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) + for i in range(0,n[0]): + for j in range(0,n[1]): + kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) + + kx = np.array(kx, dtype=dtype) + ky = np.array(ky, dtype=dtype) + kz = np.array(kz, dtype=dtype) + k = np.sqrt(np.array(kx**2+ky**2+kz**2, dtype=dtype)) + + # only use the positive frequencies + inds = np.where(np.logical_and(k**2 >= kmin**2, k**2 < (kmax+1)**2)) + nr = len(inds[0]) + + phasex = np.zeros(n, dtype=dtype) + phasex[inds] = 2.*pi*np.random.uniform(size=nr) + fx = np.zeros(n, dtype=dtype) + fx[inds] = np.random.normal(size=nr) + + phasey = np.zeros(n, dtype=dtype) + phasey[inds] = 2.*pi*np.random.uniform(size=nr) + fy = np.zeros(n, dtype=dtype) + fy[inds] = np.random.normal(size=nr) + + phasez = np.zeros(n, dtype=dtype) + phasez[inds] = 2.*pi*np.random.uniform(size=nr) + fz = np.zeros(n, dtype=dtype) + fz[inds] = np.random.normal(size=nr) + + # rescale perturbation amplitude so that low number statistics + # at low k do not throw off the desired power law scaling. + for i in range(kmin, kmax+1): + slice_inds = np.where(np.logical_and(k >= i, k < i+1)) + rescale = sqrt(np.sum(np.abs(fx[slice_inds])**2 + np.abs(fy[slice_inds])**2 + np.abs(fz[slice_inds])**2)) + fx[slice_inds] = fx[slice_inds]/rescale + fy[slice_inds] = fy[slice_inds]/rescale + fz[slice_inds] = fz[slice_inds]/rescale + + # set the power law behavior + # wave number bins + fx[inds] = fx[inds]*k[inds]**-(0.5*alpha) + fy[inds] = fy[inds]*k[inds]**-(0.5*alpha) + fz[inds] = fz[inds]*k[inds]**-(0.5*alpha) + + # add in phases + fx = np.cos(phasex)*fx + 1j*np.sin(phasex)*fx + fy = np.cos(phasey)*fy + 1j*np.sin(phasey)*fy + fz = np.cos(phasez)*fz + 1j*np.sin(phasez)*fz + + return fx, fy, fz, kx, ky, kz + + +def normalize(fx, fy, fz): + norm = np.sqrt(np.sum(fx**2 + fy**2 + fz**2)/np.product(n)) + fx = fx/norm + fy = fy/norm + fz = fz/norm + return fx, fy, fz + + +def make_perturbations(n, kmin, kmax, f_solenoidal): + fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) + if f_solenoidal != None: + k2 = kx**2+ky**2+kz**2 + # solenoidal part + fxs = 0.; fys =0.; fzs = 0. + if f_solenoidal != 0.0: + fxs = np.real(fx - kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fys = np.real(fy - ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fzs = np.real(fz - kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + ind = np.where(k2 == 0) + fxs[ind] = 0.; fys[ind] = 0.; fzs[ind] = 0. + # need to normalize this before applying relative weighting of solenoidal / compressive components + norm = np.sqrt(np.sum(fxs**2+fys**2+fzs**2)) + fxs = fxs/norm + fys = fys/norm + fzs = fzs/norm + # compressive part + # get a different random cube for the compressive part + # so that we can target the RMS solenoidal fraction, + # instead of setting a constant solenoidal fraction everywhere. + fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) + fxc = 0.; fyc =0.; fzc = 0. + if f_solenoidal != 1.0: + fxc = np.real(kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fyc = np.real(ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fzc = np.real(kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + ind = np.where(k2 == 0) + fxc[ind] = 0.; fyc[ind] = 0.; fzc[ind] = 0. + # need to normalize this before applying relative weighting of solenoidal / compressive components + norm = np.sqrt(np.sum(fxc**2+fyc**2+fzc**2)) + fxc = fxc/norm + fyc = fyc/norm + fzc = fzc/norm + # back to real space + pertx = np.real(np.fft.ifftn(f_solenoidal*fxs + (1.-f_solenoidal)*fxc)) + perty = np.real(np.fft.ifftn(f_solenoidal*fys + (1.-f_solenoidal)*fyc)) + pertz = np.real(np.fft.ifftn(f_solenoidal*fzs + (1.-f_solenoidal)*fzc)) + else: + # just convert to real space + pertx = np.real(np.fft.ifftn(fx)) + perty = np.real(np.fft.ifftn(fy)) + pertz = np.real(np.fft.ifftn(fz)) + + # subtract off COM (assuming uniform density) + pertx = pertx-np.average(pertx) + perty = perty-np.average(perty) + pertz = pertz-np.average(pertz) + # scale RMS of perturbation cube to unity + pertx, perty, pertz = normalize(pertx, perty, pertz) + return pertx, perty, pertz + + +def cut_sphere(pertx, perty, pertz, rad): + # Make radial array + x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] + x = x - (n[0]-1)/2. + y = y - (n[1]-1)/2. + z = z - (n[2]-1)/2. + r2 = x**2+y**2+z**2 + # Get range of indices we want to set to zero, and those we want to keep + idx0 = r2 > (rad*n[0]/2.0)**2 + idx1 = np.logical_not(idx0) + # Zero outside the desired region + pertx[idx0] = 0.0 + perty[idx0] = 0.0 + pertz[idx0] = 0.0 + # Recompute COM velocity, and renormalize + pertx[idx1] = pertx[idx1]-np.average(pertx[idx1]) + perty[idx1] = perty[idx1]-np.average(perty[idx1]) + pertz[idx1] = pertz[idx1]-np.average(pertz[idx1]) + pertx, perty, pertz = normalize(pertx, perty, pertz) + return pertx, perty, pertz + + +def get_erot_ke_ratio(pertx, perty, pertz, rad=-1.0): + x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] + x = x - (n[0]-1)/2. + y = y - (n[1]-1)/2. + z = z - (n[2]-1)/2. + r2 = x**2+y**2+z**2 + if rad > 0: + idx0 = r2 > (rad*n[0]/2.0)**2 + r2[idx0] = 0.0 + erot_ke_ratio = (np.sum(y*pertz-z*perty)**2 + + np.sum(z*pertx-x*pertz)**2 + + np.sum(x*perty-y*pertx)**2)/(np.sum(r2)*np.product(n)) + return erot_ke_ratio + + +def plot_spectrum1D(pertx, perty, pertz): + # plot the 1D power to check the scaling. + fx = np.abs(np.fft.fftn(pertx)) + fy = np.abs(np.fft.fftn(perty)) + fz = np.abs(np.fft.fftn(pertz)) + fx = np.abs(fx) + fy = np.abs(fy) + fz = np.abs(fz) + kx = np.zeros(n, dtype=dtype) + ky = np.zeros(n, dtype=dtype) + kz = np.zeros(n, dtype=dtype) + # perform fft k-ordering convention shifts + for j in range(0,n[1]): + for k in range(0,n[2]): + kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) + for i in range(0,n[0]): + for k in range(0,n[2]): + ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) + for i in range(0,n[0]): + for j in range(0,n[1]): + kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) + k = np.sqrt(np.array(kx**2+ky**2+kz**2,dtype=dtype)) + k1d = [] + power = [] + for i in range(kmin,kmax+1): + slice_inds = np.where(np.logical_and(k >= i, k < i+1)) + k1d.append(i+0.5) + power.append(np.sum(fx[slice_inds]**2 + fy[slice_inds]**2 + fz[slice_inds]**2)) + print(i,power[-1]) + import matplotlib.pyplot as plt + plt.loglog(k1d, power) + plt.show() + + +################### +# input parameters, read from command line +################### +parser = OptionParser() +parser.add_option('--kmin', dest='kmin', + help='minimum wavenumber.', + default=-1) +parser.add_option('--kmax', dest='kmax', + help='maximum wavenumber.', + default=-1) +parser.add_option('--size', dest='size', + help='size of each direction of data cube. default=256', + default=256) +parser.add_option('--alpha', dest='alpha', + help='negative of power law slope. (Power ~ k^-alpha) '+ + 'supersonic turbulence is near alpha=2. '+ + 'driving over a narrow band of two modes is often done with alpha=0', + default = None) +parser.add_option('--seed', dest='seed', + help='seed for random # generation. default=0', + default = 0) +parser.add_option('--f_solenoidal', dest='f_solenoidal', + help='volume RMS fraction of solenoidal component of the perturbations relative to the total. ' + + 'If --f_solenoidal=None, the motions are purely random. For low wave numbers ' + + 'the relative imporance of solenoidal to compressive may be sensitive to the ' + + 'choice of radom seed. It has been suggested (Federrath 2008) that ' + + 'f_solenoidal=2/3 is the most natural driving mode and this is currently' + + 'the suggested best-practice.', + default = -1) +parser.add_option('--sphererad', dest='rad', + help='if set, perturbations are set to zero outside spherical region, '+ + 'and the perturbation field is shifted and renormalized to keep the '+ + 'center of mass velocity at zero and the variance at unity; the '+ + 'spherical region cut out is centered at the center of the perturbation '+ + 'cube, and has a radius given by the value of this parameter, with sphererad = '+ + '1 corresponding to the spherical region going all the way to the edge of the '+ + 'perturbation cube', + default = -1.0) + +(options, args) = parser.parse_args() + +# size of the data domain +n = [int(options.size), int(options.size), int(options.size)] +# range of perturbation length scale in units of the smallest side of the domain +kmin = int(options.kmin) +kmax = int(options.kmax) +print(kmin, kmax) +if kmin > kmax or kmin < 0 or kmax < 0: + print("kmin must be < kmax, with kmin > 0, kmax > 0. See --help.") + sys.exit(0) +if kmax > floor(np.min(n))/2: + print("kmax must be <= floor(size/2). See --help.") + sys.exit(0) +f_solenoidal = options.f_solenoidal +if f_solenoidal == "None" or f_solenoidal == "none": + f_solenoidal = None +else: + f_solenoidal = float(options.f_solenoidal) + if f_solenoidal > 1. or f_solenoidal < 0.: + print("You must choose f_solenoidal. See --help.") + sys.exit(0) +alpha = options.alpha +if alpha==None: + print("You must choose a power law slope, alpha. See --help.") + sys.exit(0) +alpha = float(options.alpha) +if alpha < 0.: + print("alpha is less than zero. That's probably not what you want. See --help.") + sys.exit(0) +seed = int(options.seed) +# data precision +dtype = np.float64 +# ratio of solenoidal to compressive components +if options.f_solenoidal=="None" or options.f_solenoidal==None: + f_solenoidal = None +else: + f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) +rad = float(options.rad) +if rad > 1: + raise ValueError('sphererad is '+options.rad+', must be from 0 to 1') + +################### +# begin computation +################### + +np.random.seed(seed=seed) +pertx, perty, pertz = make_perturbations(n, kmin, kmax, f_solenoidal) +if rad > 0: + pertx, perty, pertz = cut_sphere(pertx, perty, pertz, rad) +erot_ke_ratio = get_erot_ke_ratio(pertx, perty, pertz, rad) +print("erot_ke_ratio = ", erot_ke_ratio) + +# hdf5 output +f = h5py.File('zdrv.hdf5', 'w') +ds = f['/'].create_dataset('pertx', n, dtype=np.float64) +ds[:] = pertx +ds = f['/'].create_dataset('perty', n, dtype=np.float64) +ds[:] = perty +ds = f['/'].create_dataset('pertz', n, dtype=np.float64) +ds[:] = pertz +f['/'].attrs['kmin'] = kmin +f['/'].attrs['kmax'] = kmax +f['/'].attrs['alpha'] = alpha +if f_solenoidal!=None: f['/'].attrs['f_solenoidal'] = f_solenoidal +if rad > 0: f['/'].attrs['sphererad'] = rad +f['/'].attrs['erot_ke_ratio'] = erot_ke_ratio +f['/'].attrs['seed'] = seed +f.close() + From 9427d61b70770d8d844e22d4172fbd6076e2ec40 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:48:25 +0200 Subject: [PATCH 030/165] move perturb to src --- src/PopIII/perturbation.py | 311 ------------------------------------- 1 file changed, 311 deletions(-) delete mode 100644 src/PopIII/perturbation.py diff --git a/src/PopIII/perturbation.py b/src/PopIII/perturbation.py deleted file mode 100644 index d6d53e640..000000000 --- a/src/PopIII/perturbation.py +++ /dev/null @@ -1,311 +0,0 @@ -import h5py -import numpy as np -import numpy.fft -from math import * -from optparse import OptionParser -import sys - - -def init_perturbations(n, kmin, kmax, dtype): - kx = np.zeros(n, dtype=dtype) - ky = np.zeros(n, dtype=dtype) - kz = np.zeros(n, dtype=dtype) - # perform fft k-ordering convention shifts - for j in range(0,n[1]): - for k in range(0,n[2]): - kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) - for i in range(0,n[0]): - for k in range(0,n[2]): - ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) - for i in range(0,n[0]): - for j in range(0,n[1]): - kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) - - kx = np.array(kx, dtype=dtype) - ky = np.array(ky, dtype=dtype) - kz = np.array(kz, dtype=dtype) - k = np.sqrt(np.array(kx**2+ky**2+kz**2, dtype=dtype)) - - # only use the positive frequencies - inds = np.where(np.logical_and(k**2 >= kmin**2, k**2 < (kmax+1)**2)) - nr = len(inds[0]) - - phasex = np.zeros(n, dtype=dtype) - phasex[inds] = 2.*pi*np.random.uniform(size=nr) - fx = np.zeros(n, dtype=dtype) - fx[inds] = np.random.normal(size=nr) - - phasey = np.zeros(n, dtype=dtype) - phasey[inds] = 2.*pi*np.random.uniform(size=nr) - fy = np.zeros(n, dtype=dtype) - fy[inds] = np.random.normal(size=nr) - - phasez = np.zeros(n, dtype=dtype) - phasez[inds] = 2.*pi*np.random.uniform(size=nr) - fz = np.zeros(n, dtype=dtype) - fz[inds] = np.random.normal(size=nr) - - # rescale perturbation amplitude so that low number statistics - # at low k do not throw off the desired power law scaling. - for i in range(kmin, kmax+1): - slice_inds = np.where(np.logical_and(k >= i, k < i+1)) - rescale = sqrt(np.sum(np.abs(fx[slice_inds])**2 + np.abs(fy[slice_inds])**2 + np.abs(fz[slice_inds])**2)) - fx[slice_inds] = fx[slice_inds]/rescale - fy[slice_inds] = fy[slice_inds]/rescale - fz[slice_inds] = fz[slice_inds]/rescale - - # set the power law behavior - # wave number bins - fx[inds] = fx[inds]*k[inds]**-(0.5*alpha) - fy[inds] = fy[inds]*k[inds]**-(0.5*alpha) - fz[inds] = fz[inds]*k[inds]**-(0.5*alpha) - - # add in phases - fx = np.cos(phasex)*fx + 1j*np.sin(phasex)*fx - fy = np.cos(phasey)*fy + 1j*np.sin(phasey)*fy - fz = np.cos(phasez)*fz + 1j*np.sin(phasez)*fz - - return fx, fy, fz, kx, ky, kz - - -def normalize(fx, fy, fz): - norm = np.sqrt(np.sum(fx**2 + fy**2 + fz**2)/np.product(n)) - fx = fx/norm - fy = fy/norm - fz = fz/norm - return fx, fy, fz - - -def make_perturbations(n, kmin, kmax, f_solenoidal): - fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) - if f_solenoidal != None: - k2 = kx**2+ky**2+kz**2 - # solenoidal part - fxs = 0.; fys =0.; fzs = 0. - if f_solenoidal != 0.0: - fxs = np.real(fx - kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fys = np.real(fy - ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fzs = np.real(fz - kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - ind = np.where(k2 == 0) - fxs[ind] = 0.; fys[ind] = 0.; fzs[ind] = 0. - # need to normalize this before applying relative weighting of solenoidal / compressive components - norm = np.sqrt(np.sum(fxs**2+fys**2+fzs**2)) - fxs = fxs/norm - fys = fys/norm - fzs = fzs/norm - # compressive part - # get a different random cube for the compressive part - # so that we can target the RMS solenoidal fraction, - # instead of setting a constant solenoidal fraction everywhere. - fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) - fxc = 0.; fyc =0.; fzc = 0. - if f_solenoidal != 1.0: - fxc = np.real(kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fyc = np.real(ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fzc = np.real(kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - ind = np.where(k2 == 0) - fxc[ind] = 0.; fyc[ind] = 0.; fzc[ind] = 0. - # need to normalize this before applying relative weighting of solenoidal / compressive components - norm = np.sqrt(np.sum(fxc**2+fyc**2+fzc**2)) - fxc = fxc/norm - fyc = fyc/norm - fzc = fzc/norm - # back to real space - pertx = np.real(np.fft.ifftn(f_solenoidal*fxs + (1.-f_solenoidal)*fxc)) - perty = np.real(np.fft.ifftn(f_solenoidal*fys + (1.-f_solenoidal)*fyc)) - pertz = np.real(np.fft.ifftn(f_solenoidal*fzs + (1.-f_solenoidal)*fzc)) - else: - # just convert to real space - pertx = np.real(np.fft.ifftn(fx)) - perty = np.real(np.fft.ifftn(fy)) - pertz = np.real(np.fft.ifftn(fz)) - - # subtract off COM (assuming uniform density) - pertx = pertx-np.average(pertx) - perty = perty-np.average(perty) - pertz = pertz-np.average(pertz) - # scale RMS of perturbation cube to unity - pertx, perty, pertz = normalize(pertx, perty, pertz) - return pertx, perty, pertz - - -def cut_sphere(pertx, perty, pertz, rad): - # Make radial array - x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] - x = x - (n[0]-1)/2. - y = y - (n[1]-1)/2. - z = z - (n[2]-1)/2. - r2 = x**2+y**2+z**2 - # Get range of indices we want to set to zero, and those we want to keep - idx0 = r2 > (rad*n[0]/2.0)**2 - idx1 = np.logical_not(idx0) - # Zero outside the desired region - pertx[idx0] = 0.0 - perty[idx0] = 0.0 - pertz[idx0] = 0.0 - # Recompute COM velocity, and renormalize - pertx[idx1] = pertx[idx1]-np.average(pertx[idx1]) - perty[idx1] = perty[idx1]-np.average(perty[idx1]) - pertz[idx1] = pertz[idx1]-np.average(pertz[idx1]) - pertx, perty, pertz = normalize(pertx, perty, pertz) - return pertx, perty, pertz - - -def get_erot_ke_ratio(pertx, perty, pertz, rad=-1.0): - x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] - x = x - (n[0]-1)/2. - y = y - (n[1]-1)/2. - z = z - (n[2]-1)/2. - r2 = x**2+y**2+z**2 - if rad > 0: - idx0 = r2 > (rad*n[0]/2.0)**2 - r2[idx0] = 0.0 - erot_ke_ratio = (np.sum(y*pertz-z*perty)**2 + - np.sum(z*pertx-x*pertz)**2 + - np.sum(x*perty-y*pertx)**2)/(np.sum(r2)*np.product(n)) - return erot_ke_ratio - - -def plot_spectrum1D(pertx, perty, pertz): - # plot the 1D power to check the scaling. - fx = np.abs(np.fft.fftn(pertx)) - fy = np.abs(np.fft.fftn(perty)) - fz = np.abs(np.fft.fftn(pertz)) - fx = np.abs(fx) - fy = np.abs(fy) - fz = np.abs(fz) - kx = np.zeros(n, dtype=dtype) - ky = np.zeros(n, dtype=dtype) - kz = np.zeros(n, dtype=dtype) - # perform fft k-ordering convention shifts - for j in range(0,n[1]): - for k in range(0,n[2]): - kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) - for i in range(0,n[0]): - for k in range(0,n[2]): - ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) - for i in range(0,n[0]): - for j in range(0,n[1]): - kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) - k = np.sqrt(np.array(kx**2+ky**2+kz**2,dtype=dtype)) - k1d = [] - power = [] - for i in range(kmin,kmax+1): - slice_inds = np.where(np.logical_and(k >= i, k < i+1)) - k1d.append(i+0.5) - power.append(np.sum(fx[slice_inds]**2 + fy[slice_inds]**2 + fz[slice_inds]**2)) - print(i,power[-1]) - import matplotlib.pyplot as plt - plt.loglog(k1d, power) - plt.show() - - -################### -# input parameters, read from command line -################### -parser = OptionParser() -parser.add_option('--kmin', dest='kmin', - help='minimum wavenumber.', - default=-1) -parser.add_option('--kmax', dest='kmax', - help='maximum wavenumber.', - default=-1) -parser.add_option('--size', dest='size', - help='size of each direction of data cube. default=256', - default=256) -parser.add_option('--alpha', dest='alpha', - help='negative of power law slope. (Power ~ k^-alpha) '+ - 'supersonic turbulence is near alpha=2. '+ - 'driving over a narrow band of two modes is often done with alpha=0', - default = None) -parser.add_option('--seed', dest='seed', - help='seed for random # generation. default=0', - default = 0) -parser.add_option('--f_solenoidal', dest='f_solenoidal', - help='volume RMS fraction of solenoidal component of the perturbations relative to the total. ' + - 'If --f_solenoidal=None, the motions are purely random. For low wave numbers ' + - 'the relative imporance of solenoidal to compressive may be sensitive to the ' + - 'choice of radom seed. It has been suggested (Federrath 2008) that ' + - 'f_solenoidal=2/3 is the most natural driving mode and this is currently' + - 'the suggested best-practice.', - default = -1) -parser.add_option('--sphererad', dest='rad', - help='if set, perturbations are set to zero outside spherical region, '+ - 'and the perturbation field is shifted and renormalized to keep the '+ - 'center of mass velocity at zero and the variance at unity; the '+ - 'spherical region cut out is centered at the center of the perturbation '+ - 'cube, and has a radius given by the value of this parameter, with sphererad = '+ - '1 corresponding to the spherical region going all the way to the edge of the '+ - 'perturbation cube', - default = -1.0) - -(options, args) = parser.parse_args() - -# size of the data domain -n = [int(options.size), int(options.size), int(options.size)] -# range of perturbation length scale in units of the smallest side of the domain -kmin = int(options.kmin) -kmax = int(options.kmax) -print(kmin, kmax) -if kmin > kmax or kmin < 0 or kmax < 0: - print("kmin must be < kmax, with kmin > 0, kmax > 0. See --help.") - sys.exit(0) -if kmax > floor(np.min(n))/2: - print("kmax must be <= floor(size/2). See --help.") - sys.exit(0) -f_solenoidal = options.f_solenoidal -if f_solenoidal == "None" or f_solenoidal == "none": - f_solenoidal = None -else: - f_solenoidal = float(options.f_solenoidal) - if f_solenoidal > 1. or f_solenoidal < 0.: - print("You must choose f_solenoidal. See --help.") - sys.exit(0) -alpha = options.alpha -if alpha==None: - print("You must choose a power law slope, alpha. See --help.") - sys.exit(0) -alpha = float(options.alpha) -if alpha < 0.: - print("alpha is less than zero. That's probably not what you want. See --help.") - sys.exit(0) -seed = int(options.seed) -# data precision -dtype = np.float64 -# ratio of solenoidal to compressive components -if options.f_solenoidal=="None" or options.f_solenoidal==None: - f_solenoidal = None -else: - f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) -rad = float(options.rad) -if rad > 1: - raise ValueError('sphererad is '+options.rad+', must be from 0 to 1') - -################### -# begin computation -################### - -np.random.seed(seed=seed) -pertx, perty, pertz = make_perturbations(n, kmin, kmax, f_solenoidal) -if rad > 0: - pertx, perty, pertz = cut_sphere(pertx, perty, pertz, rad) -erot_ke_ratio = get_erot_ke_ratio(pertx, perty, pertz, rad) -print("erot_ke_ratio = ", erot_ke_ratio) - -# hdf5 output -f = h5py.File('zdrv.hdf5', 'w') -ds = f['/'].create_dataset('pertx', n, dtype=np.float64) -ds[:] = pertx -ds = f['/'].create_dataset('perty', n, dtype=np.float64) -ds[:] = perty -ds = f['/'].create_dataset('pertz', n, dtype=np.float64) -ds[:] = pertz -f['/'].attrs['kmin'] = kmin -f['/'].attrs['kmax'] = kmax -f['/'].attrs['alpha'] = alpha -if f_solenoidal!=None: f['/'].attrs['f_solenoidal'] = f_solenoidal -if rad > 0: f['/'].attrs['sphererad'] = rad -f['/'].attrs['erot_ke_ratio'] = erot_ke_ratio -f['/'].attrs['seed'] = seed -f.close() - From b912d38172bcdf1a509badbf09b7709671a46761 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:49:55 +0200 Subject: [PATCH 031/165] moved perturb to src --- src/PopIII/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index b6d3c6a6c..089ce1bab 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -67,7 +67,7 @@ if (AMReX_SPACEDIM EQUAL 3) endif() - add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=2 --f_solenoidal=2./3. WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) add_test(NAME PopIII COMMAND popiii PopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP PopIII_fixture) set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED PopIII_fixture) From 5669ea7265875c30975bf49444b04f2069090562 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:50:15 +0200 Subject: [PATCH 032/165] fix bug --- src/StarCluster/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StarCluster/CMakeLists.txt b/src/StarCluster/CMakeLists.txt index 796f1f1c8..f41149970 100644 --- a/src/StarCluster/CMakeLists.txt +++ b/src/StarCluster/CMakeLists.txt @@ -1,7 +1,7 @@ if (AMReX_SPACEDIM EQUAL 3) # Define a custom target that runs the Python script to produce the input perturbations file - add_executable(star_cluster star_cluster.cpp ../TurbDataReader.cpp ${QuokkaObjSources}) + add_executable(star_cluster star_cluster.cpp TurbDataReader.cpp ${QuokkaObjSources}) if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(star_cluster) endif() From dcae899d29e673334f644de194c632270a697d78 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 21:55:28 +0200 Subject: [PATCH 033/165] re-add enabled --- src/PopIII/popiii.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 4274b67be..b16533175 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -48,6 +48,7 @@ template <> struct HydroSystem_Traits { template <> struct Physics_Traits { // cell-centred static constexpr bool is_hydro_enabled = true; + static constexpr bool is_chemistry_enabled = false; // in the future, this could point to microphysics, and set to true static constexpr int numMassScalars = NumSpec; // number of chemical species static constexpr int numPassiveScalars = numMassScalars + 0; // we only have mass scalars static constexpr bool is_radiation_enabled = false; From 2b87ce806c4f74c8af7941671c93626b22c75bcd Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 22:27:17 +0200 Subject: [PATCH 034/165] updated params to match PopIII expectations --- src/PopIII/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index 089ce1bab..030dd82b0 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -67,7 +67,7 @@ if (AMReX_SPACEDIM EQUAL 3) endif() - add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=2 --f_solenoidal=2./3. WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=1.8 --f_solenoidal=0.66667 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) add_test(NAME PopIII COMMAND popiii PopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP PopIII_fixture) set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED PopIII_fixture) From c564972009654131d9be12709fa80c73a6c514eb Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 23:18:08 +0200 Subject: [PATCH 035/165] remove turbdatareader --- src/StarCluster/TurbDataReader.cpp | 119 ----------------------------- src/StarCluster/TurbDataReader.hpp | 59 -------------- 2 files changed, 178 deletions(-) delete mode 100644 src/StarCluster/TurbDataReader.cpp delete mode 100644 src/StarCluster/TurbDataReader.hpp diff --git a/src/StarCluster/TurbDataReader.cpp b/src/StarCluster/TurbDataReader.cpp deleted file mode 100644 index 6834c17a1..000000000 --- a/src/StarCluster/TurbDataReader.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file TurbDataReader.cpp -/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. -/// - -#include "TurbDataReader.hpp" -#include "AMReX_Arena.H" -#include "AMReX_BLassert.H" -#include "AMReX_Print.H" -#include "AMReX_TableData.H" -#include - -auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D -{ - // open dataset - hid_t dset_id = 0; - dset_id = H5Dopen2(file_id, dataset_name, H5P_DEFAULT); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dset_id != -1, "Can't open table!"); - - // get dimensions - hid_t const dspace = H5Dget_space(dset_id); - const int ndims = H5Sget_simple_extent_ndims(dspace); - std::vector dims(ndims); - H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); - - uint64_t data_size = 1; - for (int idim = 0; idim < ndims; ++idim) { - data_size *= dims[idim]; - } - - // allocate array for dataset storage - auto *temp_data = new double[data_size]; - - // read dataset - herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); - - // close dataset - status = H5Dclose(dset_id); - - // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 - // tables use row-major (C-order) indexing! - amrex::GpuArray const lo{0, 0, 0}; - amrex::GpuArray const hi{static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])}; - auto table = amrex::Table3D(temp_data, lo, hi); - return table; -} - -void initialize_turbdata(turb_data &data, std::string &data_file) -{ - amrex::Print() << "Initializing turbulence data...\n"; - amrex::Print() << fmt::format("data_file: {}.\n", data_file); - - herr_t status = 0; - herr_t const h5_error = -1; - - // open file - hid_t file_id = 0; - file_id = H5Fopen(data_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(file_id != h5_error, "Failed to open data file!"); - - data.dvx = read_dataset(file_id, "pertx"); - data.dvy = read_dataset(file_id, "perty"); - data.dvz = read_dataset(file_id, "pertz"); - - // close file - status = H5Fclose(file_id); -} - -auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData -{ - amrex::Array tlo{in_t.begin[0], in_t.begin[1], in_t.begin[2]}; - amrex::Array thi{in_t.end[0] - 1, in_t.end[1] - 1, in_t.end[2] - 1}; - amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); - auto h_table = tableData.table(); - - amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; - - // fill tableData - for (int i = tlo[0]; i <= thi[0]; ++i) { - for (int j = tlo[1]; j <= thi[1]; ++j) { - for (int k = tlo[2]; k <= thi[2]; ++k) { - h_table(i, j, k) = in_t(i, j, k); - } - } - } - - return tableData; -} - -auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real -{ - amrex::Array tlo = dvx.lo(); - amrex::Array thi = dvx.hi(); - auto const &dvx_table = dvx.const_table(); - auto const &dvy_table = dvy.const_table(); - auto const &dvz_table = dvz.const_table(); - - // compute rms power - amrex::Real rms_sq = 0; - amrex::Long N = 0; - for (int i = tlo[0]; i <= thi[0]; ++i) { - for (int j = tlo[1]; j <= thi[1]; ++j) { - for (int k = tlo[2]; k <= thi[2]; ++k) { - amrex::Real const vx = dvx_table(i, j, k); - amrex::Real const vy = dvy_table(i, j, k); - amrex::Real const vz = dvz_table(i, j, k); - rms_sq += vx * vx + vy * vy + vz * vz; - ++N; - } - } - } - rms_sq /= static_cast(N); - return std::sqrt(rms_sq); -} diff --git a/src/StarCluster/TurbDataReader.hpp b/src/StarCluster/TurbDataReader.hpp deleted file mode 100644 index b127cc92a..000000000 --- a/src/StarCluster/TurbDataReader.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef TURBDATAREADER_HPP_ // NOLINT -#define TURBDATAREADER_HPP_ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file TurbDataReader.hpp -/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. -/// - -#include -#include -#include -#include -#include - -#include "fmt/core.h" -#include -#include -#include - -#include "AMReX.H" -#include "AMReX_Array.H" -#include "AMReX_BLassert.H" -#include "AMReX_TableData.H" - -/* HDF5 definitions */ - -#define HDF5_FILE_I4 H5T_STD_I32BE -#define HDF5_FILE_I8 H5T_STD_I64BE -#define HDF5_FILE_R4 H5T_IEEE_F32BE -#define HDF5_FILE_R8 H5T_IEEE_F64BE -#define HDF5_FILE_B8 H5T_STD_B8BE - -#define HDF5_I4 H5T_NATIVE_INT -#define HDF5_I8 H5T_NATIVE_LLONG -#define HDF5_R4 H5T_NATIVE_FLOAT -#define HDF5_R8 H5T_NATIVE_DOUBLE -#define HDF5_R16 H5T_NATIVE_LDOUBLE - -// Cooling table storage - -using turb_data = struct turb_data { - // values - amrex::Table3D dvx; - amrex::Table3D dvy; - amrex::Table3D dvz; -}; - -void initialize_turbdata(turb_data &data, std::string &data_file); - -auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D; - -auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData; - -auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real; - -#endif // TURBDATAREADER_HPP_ From 930770573fe9370d11b6ded0e6628aca906f3a51 Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 23:18:40 +0200 Subject: [PATCH 036/165] remove turbdatareader --- src/PopIII/TurbDataReader.cpp | 119 ---------------------------------- src/PopIII/TurbDataReader.hpp | 59 ----------------- 2 files changed, 178 deletions(-) delete mode 100644 src/PopIII/TurbDataReader.cpp delete mode 100644 src/PopIII/TurbDataReader.hpp diff --git a/src/PopIII/TurbDataReader.cpp b/src/PopIII/TurbDataReader.cpp deleted file mode 100644 index 6834c17a1..000000000 --- a/src/PopIII/TurbDataReader.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file TurbDataReader.cpp -/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. -/// - -#include "TurbDataReader.hpp" -#include "AMReX_Arena.H" -#include "AMReX_BLassert.H" -#include "AMReX_Print.H" -#include "AMReX_TableData.H" -#include - -auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D -{ - // open dataset - hid_t dset_id = 0; - dset_id = H5Dopen2(file_id, dataset_name, H5P_DEFAULT); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dset_id != -1, "Can't open table!"); - - // get dimensions - hid_t const dspace = H5Dget_space(dset_id); - const int ndims = H5Sget_simple_extent_ndims(dspace); - std::vector dims(ndims); - H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); - - uint64_t data_size = 1; - for (int idim = 0; idim < ndims; ++idim) { - data_size *= dims[idim]; - } - - // allocate array for dataset storage - auto *temp_data = new double[data_size]; - - // read dataset - herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); - - // close dataset - status = H5Dclose(dset_id); - - // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 - // tables use row-major (C-order) indexing! - amrex::GpuArray const lo{0, 0, 0}; - amrex::GpuArray const hi{static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])}; - auto table = amrex::Table3D(temp_data, lo, hi); - return table; -} - -void initialize_turbdata(turb_data &data, std::string &data_file) -{ - amrex::Print() << "Initializing turbulence data...\n"; - amrex::Print() << fmt::format("data_file: {}.\n", data_file); - - herr_t status = 0; - herr_t const h5_error = -1; - - // open file - hid_t file_id = 0; - file_id = H5Fopen(data_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(file_id != h5_error, "Failed to open data file!"); - - data.dvx = read_dataset(file_id, "pertx"); - data.dvy = read_dataset(file_id, "perty"); - data.dvz = read_dataset(file_id, "pertz"); - - // close file - status = H5Fclose(file_id); -} - -auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData -{ - amrex::Array tlo{in_t.begin[0], in_t.begin[1], in_t.begin[2]}; - amrex::Array thi{in_t.end[0] - 1, in_t.end[1] - 1, in_t.end[2] - 1}; - amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); - auto h_table = tableData.table(); - - amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; - - // fill tableData - for (int i = tlo[0]; i <= thi[0]; ++i) { - for (int j = tlo[1]; j <= thi[1]; ++j) { - for (int k = tlo[2]; k <= thi[2]; ++k) { - h_table(i, j, k) = in_t(i, j, k); - } - } - } - - return tableData; -} - -auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real -{ - amrex::Array tlo = dvx.lo(); - amrex::Array thi = dvx.hi(); - auto const &dvx_table = dvx.const_table(); - auto const &dvy_table = dvy.const_table(); - auto const &dvz_table = dvz.const_table(); - - // compute rms power - amrex::Real rms_sq = 0; - amrex::Long N = 0; - for (int i = tlo[0]; i <= thi[0]; ++i) { - for (int j = tlo[1]; j <= thi[1]; ++j) { - for (int k = tlo[2]; k <= thi[2]; ++k) { - amrex::Real const vx = dvx_table(i, j, k); - amrex::Real const vy = dvy_table(i, j, k); - amrex::Real const vz = dvz_table(i, j, k); - rms_sq += vx * vx + vy * vy + vz * vz; - ++N; - } - } - } - rms_sq /= static_cast(N); - return std::sqrt(rms_sq); -} diff --git a/src/PopIII/TurbDataReader.hpp b/src/PopIII/TurbDataReader.hpp deleted file mode 100644 index b127cc92a..000000000 --- a/src/PopIII/TurbDataReader.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef TURBDATAREADER_HPP_ // NOLINT -#define TURBDATAREADER_HPP_ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file TurbDataReader.hpp -/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. -/// - -#include -#include -#include -#include -#include - -#include "fmt/core.h" -#include -#include -#include - -#include "AMReX.H" -#include "AMReX_Array.H" -#include "AMReX_BLassert.H" -#include "AMReX_TableData.H" - -/* HDF5 definitions */ - -#define HDF5_FILE_I4 H5T_STD_I32BE -#define HDF5_FILE_I8 H5T_STD_I64BE -#define HDF5_FILE_R4 H5T_IEEE_F32BE -#define HDF5_FILE_R8 H5T_IEEE_F64BE -#define HDF5_FILE_B8 H5T_STD_B8BE - -#define HDF5_I4 H5T_NATIVE_INT -#define HDF5_I8 H5T_NATIVE_LLONG -#define HDF5_R4 H5T_NATIVE_FLOAT -#define HDF5_R8 H5T_NATIVE_DOUBLE -#define HDF5_R16 H5T_NATIVE_LDOUBLE - -// Cooling table storage - -using turb_data = struct turb_data { - // values - amrex::Table3D dvx; - amrex::Table3D dvy; - amrex::Table3D dvz; -}; - -void initialize_turbdata(turb_data &data, std::string &data_file); - -auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D; - -auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData; - -auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real; - -#endif // TURBDATAREADER_HPP_ From d0fa85dfa19ba16f19b5e2af7e7d97b5e636398b Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 8 Sep 2023 23:19:17 +0200 Subject: [PATCH 037/165] move turbdatareader to src --- src/PopIII/CMakeLists.txt | 2 +- src/StarCluster/CMakeLists.txt | 2 +- src/TurbDataReader.cpp | 119 +++++++++++++++++++++++++++++++++ src/TurbDataReader.hpp | 59 ++++++++++++++++ 4 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 src/TurbDataReader.cpp create mode 100644 src/TurbDataReader.hpp diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index 030dd82b0..dfbe99112 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -9,7 +9,7 @@ if (AMReX_SPACEDIM EQUAL 3) #this is critical to ensure the correct Microphysics files are linked to primordial chem target include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H") - add_executable(popiii popiii.cpp TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) + add_executable(popiii popiii.cpp ../TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM if(AMReX_GPU_BACKEND MATCHES "CUDA") diff --git a/src/StarCluster/CMakeLists.txt b/src/StarCluster/CMakeLists.txt index f41149970..796f1f1c8 100644 --- a/src/StarCluster/CMakeLists.txt +++ b/src/StarCluster/CMakeLists.txt @@ -1,7 +1,7 @@ if (AMReX_SPACEDIM EQUAL 3) # Define a custom target that runs the Python script to produce the input perturbations file - add_executable(star_cluster star_cluster.cpp TurbDataReader.cpp ${QuokkaObjSources}) + add_executable(star_cluster star_cluster.cpp ../TurbDataReader.cpp ${QuokkaObjSources}) if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(star_cluster) endif() diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp new file mode 100644 index 000000000..6834c17a1 --- /dev/null +++ b/src/TurbDataReader.cpp @@ -0,0 +1,119 @@ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file TurbDataReader.cpp +/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. +/// + +#include "TurbDataReader.hpp" +#include "AMReX_Arena.H" +#include "AMReX_BLassert.H" +#include "AMReX_Print.H" +#include "AMReX_TableData.H" +#include + +auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D +{ + // open dataset + hid_t dset_id = 0; + dset_id = H5Dopen2(file_id, dataset_name, H5P_DEFAULT); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dset_id != -1, "Can't open table!"); + + // get dimensions + hid_t const dspace = H5Dget_space(dset_id); + const int ndims = H5Sget_simple_extent_ndims(dspace); + std::vector dims(ndims); + H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); + + uint64_t data_size = 1; + for (int idim = 0; idim < ndims; ++idim) { + data_size *= dims[idim]; + } + + // allocate array for dataset storage + auto *temp_data = new double[data_size]; + + // read dataset + herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); + + // close dataset + status = H5Dclose(dset_id); + + // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 + // tables use row-major (C-order) indexing! + amrex::GpuArray const lo{0, 0, 0}; + amrex::GpuArray const hi{static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])}; + auto table = amrex::Table3D(temp_data, lo, hi); + return table; +} + +void initialize_turbdata(turb_data &data, std::string &data_file) +{ + amrex::Print() << "Initializing turbulence data...\n"; + amrex::Print() << fmt::format("data_file: {}.\n", data_file); + + herr_t status = 0; + herr_t const h5_error = -1; + + // open file + hid_t file_id = 0; + file_id = H5Fopen(data_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(file_id != h5_error, "Failed to open data file!"); + + data.dvx = read_dataset(file_id, "pertx"); + data.dvy = read_dataset(file_id, "perty"); + data.dvz = read_dataset(file_id, "pertz"); + + // close file + status = H5Fclose(file_id); +} + +auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData +{ + amrex::Array tlo{in_t.begin[0], in_t.begin[1], in_t.begin[2]}; + amrex::Array thi{in_t.end[0] - 1, in_t.end[1] - 1, in_t.end[2] - 1}; + amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); + auto h_table = tableData.table(); + + amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; + + // fill tableData + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + for (int k = tlo[2]; k <= thi[2]; ++k) { + h_table(i, j, k) = in_t(i, j, k); + } + } + } + + return tableData; +} + +auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real +{ + amrex::Array tlo = dvx.lo(); + amrex::Array thi = dvx.hi(); + auto const &dvx_table = dvx.const_table(); + auto const &dvy_table = dvy.const_table(); + auto const &dvz_table = dvz.const_table(); + + // compute rms power + amrex::Real rms_sq = 0; + amrex::Long N = 0; + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + for (int k = tlo[2]; k <= thi[2]; ++k) { + amrex::Real const vx = dvx_table(i, j, k); + amrex::Real const vy = dvy_table(i, j, k); + amrex::Real const vz = dvz_table(i, j, k); + rms_sq += vx * vx + vy * vy + vz * vz; + ++N; + } + } + } + rms_sq /= static_cast(N); + return std::sqrt(rms_sq); +} diff --git a/src/TurbDataReader.hpp b/src/TurbDataReader.hpp new file mode 100644 index 000000000..b127cc92a --- /dev/null +++ b/src/TurbDataReader.hpp @@ -0,0 +1,59 @@ +#ifndef TURBDATAREADER_HPP_ // NOLINT +#define TURBDATAREADER_HPP_ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file TurbDataReader.hpp +/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. +/// + +#include +#include +#include +#include +#include + +#include "fmt/core.h" +#include +#include +#include + +#include "AMReX.H" +#include "AMReX_Array.H" +#include "AMReX_BLassert.H" +#include "AMReX_TableData.H" + +/* HDF5 definitions */ + +#define HDF5_FILE_I4 H5T_STD_I32BE +#define HDF5_FILE_I8 H5T_STD_I64BE +#define HDF5_FILE_R4 H5T_IEEE_F32BE +#define HDF5_FILE_R8 H5T_IEEE_F64BE +#define HDF5_FILE_B8 H5T_STD_B8BE + +#define HDF5_I4 H5T_NATIVE_INT +#define HDF5_I8 H5T_NATIVE_LLONG +#define HDF5_R4 H5T_NATIVE_FLOAT +#define HDF5_R8 H5T_NATIVE_DOUBLE +#define HDF5_R16 H5T_NATIVE_LDOUBLE + +// Cooling table storage + +using turb_data = struct turb_data { + // values + amrex::Table3D dvx; + amrex::Table3D dvy; + amrex::Table3D dvz; +}; + +void initialize_turbdata(turb_data &data, std::string &data_file); + +auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D; + +auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData; + +auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real; + +#endif // TURBDATAREADER_HPP_ From 57b9827d1d2caf296bd6d953ef5496d31b7ce969 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 9 Sep 2023 06:10:03 +0200 Subject: [PATCH 038/165] dont refine outside core --- src/PopIII/popiii.cpp | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index b16533175..792f27630 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -335,34 +335,40 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: }); } + template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) { // refine on Jeans length const int N_cells = 32; // inverse of the 'Jeans number' [Truelove et al. (1997)] - const amrex::Real dx = geom[lev].CellSizeArray()[0]; const amrex::Real G = Gconst_; + const amrex::Real dx = geom[lev].CellSizeArray()[0]; - auto const &state = state_new_cc_[lev].const_arrays(); - auto tag = tags.arrays(); - amrex::Print() << "start error est " << std::endl; - amrex::ParallelFor(tags, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + for (amrex::MFIter mfi(state_new_cc_[lev]); mfi.isValid(); ++mfi) { + const amrex::Box &box = mfi.validbox(); + const auto state = state_new_cc_[lev].const_array(mfi); + const auto tag = tags.array(mfi); + const int nidx = HydroSystem::density_index; - Real const rho = state[bx](i, j, k, HydroSystem::density_index); - Real pressure = HydroSystem::ComputePressure(state[bx], i, j, k); - amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + Real const rho = state(i, j, k, nidx); + Real pressure = HydroSystem::ComputePressure(state, i, j, k); + amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state, i, j, k); - const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); + amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + + const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); + + if (l_Jeans < (N_cells * dx) && rho > 2e-20) { + tag(i, j, k) = amrex::TagBox::SET; + + } + }); + } - if (l_Jeans < (N_cells * dx)) { - tag[bx](i, j, k) = amrex::TagBox::SET; - printf("tagging cell for refinement \n"); - } - }); - amrex::Print() << "end error est " << std::endl; } + template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const { // compute derived variables and save in 'mf' From 9abec00317e537cac2549f04e06e5b059fb9d880 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 9 Sep 2023 06:28:45 +0200 Subject: [PATCH 039/165] remove is chem enabled --- src/PopIII/popiii.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 792f27630..fdcdef3c6 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -48,7 +48,6 @@ template <> struct HydroSystem_Traits { template <> struct Physics_Traits { // cell-centred static constexpr bool is_hydro_enabled = true; - static constexpr bool is_chemistry_enabled = false; // in the future, this could point to microphysics, and set to true static constexpr int numMassScalars = NumSpec; // number of chemical species static constexpr int numPassiveScalars = numMassScalars + 0; // we only have mass scalars static constexpr bool is_radiation_enabled = false; From 3a4cc582e0fd8019b992d3ed269cb6d80e76e2d2 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 9 Sep 2023 16:34:06 +0200 Subject: [PATCH 040/165] define redshift --- tests/PopIII.in | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index 8c8f25aa4..dff773dd1 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -13,7 +13,7 @@ amr.v = 1 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 128 128 128 +amr.n_cell = 64 64 64 amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this amr.max_grid_size = 128 # at least 128 for GPUs @@ -22,7 +22,7 @@ amr.grid_eff = 0.7 # default hydro.reconstruction_order = 3 # PPM cfl = 0.2 -max_timesteps = 10 +max_timesteps = 1000 stop_time = 1e16 do_reflux = 1 @@ -95,3 +95,6 @@ integrator.renormalize_abundances = 0 # tolerances integrator.rtol_spec = 1.0e-4 integrator.atol_spec = 1.0e-4 + +#assumed redshift for Pop III star formation +network.redshift = 30.0 From 6fc07b18ff91d209ab6e07f715bbff22ba1c6b15 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 9 Sep 2023 16:34:42 +0200 Subject: [PATCH 041/165] reset params --- tests/PopIII.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index dff773dd1..ba7fdd56b 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -13,7 +13,7 @@ amr.v = 1 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this amr.max_grid_size = 128 # at least 128 for GPUs @@ -22,7 +22,7 @@ amr.grid_eff = 0.7 # default hydro.reconstruction_order = 3 # PPM cfl = 0.2 -max_timesteps = 1000 +max_timesteps = 10 stop_time = 1e16 do_reflux = 1 From b9decda3404af278de63bf83ef053594ed3380df Mon Sep 17 00:00:00 2001 From: psharda Date: Wed, 13 Sep 2023 15:01:31 +0200 Subject: [PATCH 042/165] modify --- tests/PopIII.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index ba7fdd56b..52e1d5671 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -14,7 +14,7 @@ amr.v = 1 # verbosity in Amr # Resolution and refinement # ***************************************************************** amr.n_cell = 128 128 128 -amr.max_level = 0 # number of levels = max_level + 1 +amr.max_level = 10 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this amr.max_grid_size = 128 # at least 128 for GPUs amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells @@ -22,14 +22,14 @@ amr.grid_eff = 0.7 # default hydro.reconstruction_order = 3 # PPM cfl = 0.2 -max_timesteps = 10 +max_timesteps = 100000 stop_time = 1e16 do_reflux = 1 do_subcycle = 0 #ascent_interval = 10 -plotfile_interval = 20 #100 +plotfile_interval = 50 #100 checkpoint_interval = 100 perturb.cloud_radius = 3.086e18 From e9b52c945a7a040c80d757dfd1a820739cd9369c Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 13 Sep 2023 21:50:29 +0800 Subject: [PATCH 043/165] increase density floor --- src/PopIII/popiii.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index fdcdef3c6..4fd311556 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -419,12 +419,14 @@ auto problem_main() -> int // Problem initialization RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity - sim.densityFloor_ = 1e-60; + sim.densityFloor_ = 1e-25; sim.userData_.R_sphere = R_sphere; sim.userData_.numdens_init = numdens_init; sim.userData_.omega_sphere = omega_sphere; + sim.initDt_ = 1e6; + // initialize sim.setInitialConditions(); From db3d1d293ce3748e1e76c96dc3b25bb959095578 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 13 Sep 2023 16:57:18 +0200 Subject: [PATCH 044/165] return invalid state if mass scalars have negative partial densities --- src/hydro_system.hpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index bb2679a76..1ecd23afa 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -422,6 +422,18 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am const amrex::Real rho = cons(i, j, k, density_index); bool isDensityPositive = (rho > 0.); + bool isMassScalarPositive = true; + if (nmscalars_ > 0) { + amrex::GpuArray massScalars_ = RadSystem::ComputeMassScalars(cons, i, j, k); + + for (int idx = 0; idx < nmscalars_; ++idx) { + if (massScalars_[idx] < 0.0) { + isMassScalarPositive = false; + break; // Exit the loop early if any element is not positive + } + } + + } // when the dual energy method is used, we *cannot* reset on pressure // failures. on the other hand, we don't need to -- the auxiliary internal // energy is used instead! @@ -436,7 +448,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am #endif // return (isDensityPositive && isPressurePositive); - return isDensityPositive; + return isDensityPositive && isMassScalarPositive; } template From 1c7b6f2a86e85d089cad572e376feff61edf124d Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 13 Sep 2023 17:18:41 +0200 Subject: [PATCH 045/165] enforce limits on mass scalars --- src/hydro_system.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 1ecd23afa..be8e85e0d 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -29,6 +29,9 @@ #include "radiation_system.hpp" #include "valarray.hpp" +// Microphysics headers +#include "extern_parameters.H" + // this struct is specialized by the user application code // template struct HydroSystem_Traits { @@ -733,6 +736,17 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex state[bx](i, j, k, x3Momentum_index) *= rescale_factor; } + // Enforcing Limits on mass scalars + if (nmscalars_ > 0) { + + for (int idx = 0; idx < nmscalars_; ++idx) { + if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { + state[bx](i, j, k, scalar0_index + idx) = small_x * rho; + } + } + + } + // Enforcing Limits on temperature estimated from Etot and Ekin if (!HydroSystem::is_eos_isothermal()) { // recompute gas energy (to prevent P < 0) From e78afac1fc6e85022ee59d9f8c09e472b4c6fba4 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Wed, 13 Sep 2023 14:38:04 -0400 Subject: [PATCH 046/165] add LLF fallback solver --- src/LLF.hpp | 48 ++++++++++++++++++++++++++++++++++++++ src/RadhydroSimulation.hpp | 7 +++--- src/hydro_system.hpp | 20 ++++++++++++---- 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 src/LLF.hpp diff --git a/src/LLF.hpp b/src/LLF.hpp new file mode 100644 index 000000000..1d8c604ac --- /dev/null +++ b/src/LLF.hpp @@ -0,0 +1,48 @@ +#ifndef LLF_HPP_ // NOLINT +#define LLF_HPP_ + +#include "AMReX_Extension.H" +#include "AMReX_GpuQualifiers.H" +#include +#include + +#include "ArrayView.hpp" +#include "EOS.hpp" +#include "HydroState.hpp" +#include "valarray.hpp" + +namespace quokka::Riemann +{ +// Local Lax-Friedrichs (LLF) / Rusanov solver +template +AMREX_FORCE_INLINE AMREX_GPU_DEVICE auto LLF(quokka::HydroState const &sL, quokka::HydroState const &sR) + -> quokka::valarray +{ + // Toro (Eq. 10.56) + const amrex::Real Sp = std::max(std::abs(sL.u) + sL.cs, std::abs(sR.u) + sR.cs); + + /// compute fluxes + quokka::valarray U_L = {sL.rho, sL.rho * sL.u, sL.rho * sL.v, sL.rho * sL.w, sL.E, sL.Eint}; + quokka::valarray U_R = {sR.rho, sR.rho * sR.u, sR.rho * sR.v, sR.rho * sR.w, sR.E, sR.Eint}; + + // The remaining components are passive scalars, so just copy them from + // x1LeftState and x1RightState into the (left, right) state vectors U_L and + // U_R + for (int n = 0; n < N_scalars; ++n) { + const int nstart = fluxdim - N_scalars; + U_L[nstart + n] = sL.scalar[n]; + U_R[nstart + n] = sR.scalar[n]; + } + + const quokka::valarray D_L = {0., 1., 0., 0., sL.u, 0.}; + const quokka::valarray D_R = {0., 1., 0., 0., sR.u, 0.}; + const quokka::valarray F_L = sL.u * U_L + sL.P * D_L; + const quokka::valarray F_R = sR.u * U_R + sR.P * D_R; + + // open the Riemann fan + quokka::valarray F = 0.5 * (F_L + F_R) - 0.5 * Sp * (F_R - F_L); + return F; +} +} // namespace quokka::Riemann + +#endif // LLF_HPP_ diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 8fecb7e7c..42c3f649f 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1351,7 +1351,7 @@ void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &pri HydroSystem::template FlattenShocks(primVar, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); // interface-centered kernel - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); } template @@ -1401,9 +1401,8 @@ void RadhydroSimulation::hydroFOFluxFunction(amrex::MultiFab const &p { // donor-cell reconstruction HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); - - // interface-centered kernel - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + // LLF solver + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); } template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index bb2679a76..1ca35c6f5 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -25,6 +25,7 @@ #include "ArrayView.hpp" #include "EOS.hpp" #include "HLLC.hpp" +#include "LLF.hpp" #include "hyperbolic_system.hpp" #include "radiation_system.hpp" #include "valarray.hpp" @@ -36,6 +37,10 @@ template struct HydroSystem_Traits { static constexpr bool reconstruct_eint = true; }; +enum class RiemannSolver { + HLLC, LLF +}; + /// Class for the Euler equations of inviscid hydrodynamics /// template class HydroSystem : public HyperbolicSystem @@ -108,7 +113,7 @@ template class HydroSystem : public HyperbolicSystem + template static void ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, amrex::Real K_visc); @@ -853,7 +858,7 @@ template void HydroSystem::SyncDualEnergy(amrex: } template -template +template void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::MultiFab &x1FaceVel_mf, amrex::MultiFab const &x1LeftState_mf, amrex::MultiFab const &x1RightState_mf, amrex::MultiFab const &primVar_mf, const amrex::Real K_visc) { @@ -1029,8 +1034,15 @@ void HydroSystem::ComputeFluxes(amrex::MultiFab &x1Flux_mf, amrex::Mu dw = std::min(std::min(dwl, dwr), dw); #endif - // solve the Riemann problem in canonical form - quokka::valarray F_canonical = quokka::Riemann::HLLC(sL, sR, gamma_, du, dw); + // solve the Riemann problem in canonical form (i.e., where the x-dir is the normal direction) + quokka::valarray F_canonical{}; + + if constexpr (RIEMANN == RiemannSolver::HLLC) { + F_canonical = quokka::Riemann::HLLC(sL, sR, gamma_, du, dw); + } else if constexpr (RIEMANN == RiemannSolver::LLF) { + F_canonical = quokka::Riemann::LLF(sL, sR); + } + quokka::valarray F = F_canonical; // add artificial viscosity From 9150f3aace865e73458945447196fbc082f522d0 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 13 Sep 2023 22:20:29 +0200 Subject: [PATCH 047/165] use if constexpr --- src/hydro_system.hpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index be8e85e0d..edcb37be0 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -425,16 +425,16 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am const amrex::Real rho = cons(i, j, k, density_index); bool isDensityPositive = (rho > 0.); - bool isMassScalarPositive = true; - if (nmscalars_ > 0) { - amrex::GpuArray massScalars_ = RadSystem::ComputeMassScalars(cons, i, j, k); - - for (int idx = 0; idx < nmscalars_; ++idx) { - if (massScalars_[idx] < 0.0) { - isMassScalarPositive = false; - break; // Exit the loop early if any element is not positive - } - } + bool isMassScalarPositive = true; + if constexpr (nmscalars_ > 0) { + amrex::GpuArray massScalars_ = RadSystem::ComputeMassScalars(cons, i, j, k); + + for (int idx = 0; idx < nmscalars_; ++idx) { + if (massScalars_[idx] < 0.0) { + isMassScalarPositive = false; + break; // Exit the loop early if any element is not positive + } + } } // when the dual energy method is used, we *cannot* reset on pressure From 308ddf631141e7c4cfcfcd025b96a0b8870f787a Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Wed, 13 Sep 2023 16:27:44 -0400 Subject: [PATCH 048/165] fix typo --- src/LLF.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LLF.hpp b/src/LLF.hpp index 1d8c604ac..92804b316 100644 --- a/src/LLF.hpp +++ b/src/LLF.hpp @@ -40,7 +40,7 @@ AMREX_FORCE_INLINE AMREX_GPU_DEVICE auto LLF(quokka::HydroState F_R = sR.u * U_R + sR.P * D_R; // open the Riemann fan - quokka::valarray F = 0.5 * (F_L + F_R) - 0.5 * Sp * (F_R - F_L); + quokka::valarray F = 0.5 * (F_L + F_R) - 0.5 * Sp * (U_R - U_L); return F; } } // namespace quokka::Riemann From a1c6385ccab15ac5e3be4fe4be4e23998b084c3c Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 06:00:10 +0800 Subject: [PATCH 049/165] set temp floor based on redshift --- src/PopIII/popiii.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 4fd311556..016331f01 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -420,6 +420,7 @@ auto problem_main() -> int RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity sim.densityFloor_ = 1e-25; + sim.tempFloor_ = 2.73 * (30.0 + 1.0); sim.userData_.R_sphere = R_sphere; sim.userData_.numdens_init = numdens_init; From 0a4683d8843ceba422a41a1facefe21085fbac96 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 00:52:49 +0200 Subject: [PATCH 050/165] print cell coords --- src/RadhydroSimulation.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 42c3f649f..259e2c3b8 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1026,6 +1026,11 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o if (Verbose()) { amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // Calculate the coordinates based on the cell index and cell size + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; amrex::print_state(stateNew, cell_idx); } @@ -1049,6 +1054,10 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // print cell state amrex::Print() << "[FOFC-1] Flux correction failed:\n"; + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; amrex::print_state(stateNew, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; } @@ -1107,6 +1116,10 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o if (Verbose()) { amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; amrex::print_state(stateFinal, cell_idx); } @@ -1130,6 +1143,10 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // print cell state amrex::Print() << "[FOFC-2] Flux correction failed:\n"; + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; amrex::print_state(stateFinal, cell_idx); amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; } From 191c43b6126d714e36f9a77f96113c0e5eb84682 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 00:55:34 +0200 Subject: [PATCH 051/165] print cell coords --- src/RadhydroSimulation.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 259e2c3b8..4b82969c8 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1030,7 +1030,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateNew, cell_idx); } @@ -1057,7 +1057,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateNew, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; } @@ -1119,7 +1119,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateFinal, cell_idx); } @@ -1146,7 +1146,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << "): "; + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateFinal, cell_idx); amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; } From d361292bda5355b43513c0807709acb6955593ca Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 01:13:52 +0200 Subject: [PATCH 052/165] check for dimensions --- src/RadhydroSimulation.hpp | 41 ++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 4b82969c8..a42e37ccd 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1028,8 +1028,15 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // Calculate the coordinates based on the cell index and cell size amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; + if (AMReX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + if (AMReX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } + amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateNew, cell_idx); } @@ -1055,8 +1062,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // print cell state amrex::Print() << "[FOFC-1] Flux correction failed:\n"; amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; + if (AMReX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + if (AMReX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateNew, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; @@ -1117,8 +1130,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; + if (AMReX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + if (AMReX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateFinal, cell_idx); } @@ -1144,8 +1163,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // print cell state amrex::Print() << "[FOFC-2] Flux correction failed:\n"; amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - amrex::Real z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; + if (AMReX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + if (AMReX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; amrex::print_state(stateFinal, cell_idx); amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; From 08b19219f2b831b31dbdf548b4edd18eecca5cec Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 01:20:29 +0200 Subject: [PATCH 053/165] amrex spelling --- src/RadhydroSimulation.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index a42e37ccd..6316730ff 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1030,9 +1030,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = NAN; amrex::Real z_coord = NAN; - if (AMReX_SPACEDIM > 1) { + if (AMREX_SPACEDIM > 1) { y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMReX_SPACEDIM > 2) { + if (AMREX_SPACEDIM > 2) { z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); } } @@ -1064,9 +1064,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = NAN; amrex::Real z_coord = NAN; - if (AMReX_SPACEDIM > 1) { + if (AMREX_SPACEDIM > 1) { y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMReX_SPACEDIM > 2) { + if (AMREX_SPACEDIM > 2) { z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); } } @@ -1132,9 +1132,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = NAN; amrex::Real z_coord = NAN; - if (AMReX_SPACEDIM > 1) { + if (AMREX_SPACEDIM > 1) { y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMReX_SPACEDIM > 2) { + if (AMREX_SPACEDIM > 2) { z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); } } @@ -1165,9 +1165,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = NAN; amrex::Real z_coord = NAN; - if (AMReX_SPACEDIM > 1) { + if (AMREX_SPACEDIM > 1) { y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMReX_SPACEDIM > 2) { + if (AMREX_SPACEDIM > 2) { z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); } } From 0c41f67bf0aadc98fccce3799d1d84e1de9ba132 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 13:06:04 +0200 Subject: [PATCH 054/165] new function for printing coords --- src/RadhydroSimulation.hpp | 67 ++++++++++++++------------------------ 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 6316730ff..77641bd70 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -229,6 +229,8 @@ template class RadhydroSimulation : public AMRSimulation &fluxes, int nstartNew, int ncompNew) -> std::array; + void printCoordinates(const amrex::Geometry& geom, int lev, const amrex::IntVect& cell_idx); + void advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); @@ -919,6 +921,25 @@ template auto RadhydroSimulation::isCflViolated( return cflViolation; } +template +void printCoordinates(const amrex::Geometry& geom, int lev, const amrex::IntVect& cell_idx) { + + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; + + if (AMREX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + + if (AMREX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } + + amrex::Print() << "Coordinates: (" << x_coord << ", " << y_coord << ", " << z_coord << "): "; +} + + template auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, amrex::Real time, amrex::Real dt_lev) -> bool @@ -1027,17 +1048,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // Calculate the coordinates based on the cell index and cell size - amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; - if (AMREX_SPACEDIM > 1) { - y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMREX_SPACEDIM > 2) { - z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - } - } - - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; + printCoordinates(geom, lev, cell_idx); amrex::print_state(stateNew, cell_idx); } @@ -1061,17 +1072,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // print cell state amrex::Print() << "[FOFC-1] Flux correction failed:\n"; - amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; - if (AMREX_SPACEDIM > 1) { - y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMREX_SPACEDIM > 2) { - z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - } - } - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; - amrex::print_state(stateNew, cell_idx); + printCoordinates(geom, lev, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; } if (abortOnFofcFailure_ != 0) { @@ -1129,16 +1130,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o if (Verbose()) { amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; - if (AMREX_SPACEDIM > 1) { - y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMREX_SPACEDIM > 2) { - z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - } - } - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; + printCoordinates(geom, lev, cell_idx); amrex::print_state(stateFinal, cell_idx); } @@ -1162,16 +1154,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // print cell state amrex::Print() << "[FOFC-2] Flux correction failed:\n"; - amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; - if (AMREX_SPACEDIM > 1) { - y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMREX_SPACEDIM > 2) { - z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - } - } - amrex::Print() << x_coord << ", " << y_coord << ", " << z_coord << ": "; + printCoordinates(geom, lev, cell_idx); amrex::print_state(stateFinal, cell_idx); amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; } From c17048096d473f852c124b9d80d9e2c745eaabf9 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 13:18:58 +0200 Subject: [PATCH 055/165] fix error --- src/RadhydroSimulation.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 77641bd70..f6afd93d9 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -229,7 +229,7 @@ template class RadhydroSimulation : public AMRSimulation &fluxes, int nstartNew, int ncompNew) -> std::array; - void printCoordinates(const amrex::Geometry& geom, int lev, const amrex::IntVect& cell_idx); + void printCoordinates(int lev, const amrex::IntVect& cell_idx); void advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); @@ -921,8 +921,7 @@ template auto RadhydroSimulation::isCflViolated( return cflViolation; } -template -void printCoordinates(const amrex::Geometry& geom, int lev, const amrex::IntVect& cell_idx) { +template void RadhydroSimulation::printCoordinates(int lev, const amrex::IntVect& cell_idx) { amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); amrex::Real y_coord = NAN; @@ -1048,7 +1047,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // Calculate the coordinates based on the cell index and cell size - printCoordinates(geom, lev, cell_idx); + printCoordinates(lev, cell_idx); amrex::print_state(stateNew, cell_idx); } @@ -1072,7 +1071,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // print cell state amrex::Print() << "[FOFC-1] Flux correction failed:\n"; - printCoordinates(geom, lev, cell_idx); + printCoordinates(lev, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; } if (abortOnFofcFailure_ != 0) { @@ -1130,7 +1129,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o if (Verbose()) { amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - printCoordinates(geom, lev, cell_idx); + printCoordinates(lev, cell_idx); amrex::print_state(stateFinal, cell_idx); } @@ -1154,7 +1153,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o const amrex::IntVect cell_idx = redoFlag.maxIndex(0); // print cell state amrex::Print() << "[FOFC-2] Flux correction failed:\n"; - printCoordinates(geom, lev, cell_idx); + printCoordinates(lev, cell_idx); amrex::print_state(stateFinal, cell_idx); amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; } From 5c06a3cd9bfd5407981ffeddba29b0bcbe28f08a Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 13:20:28 +0200 Subject: [PATCH 056/165] declare const --- src/RadhydroSimulation.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index f6afd93d9..afdd45cb0 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -924,8 +924,8 @@ template auto RadhydroSimulation::isCflViolated( template void RadhydroSimulation::printCoordinates(int lev, const amrex::IntVect& cell_idx) { amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; + amrex::Real const y_coord = NAN; + amrex::Real const z_coord = NAN; if (AMREX_SPACEDIM > 1) { y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); From ae85251ebd424e9d5eaa12906858144e3ae4a74a Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 13:23:07 +0200 Subject: [PATCH 057/165] fix bug --- src/RadhydroSimulation.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index afdd45cb0..673ca16e3 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1072,6 +1072,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // print cell state amrex::Print() << "[FOFC-1] Flux correction failed:\n"; printCoordinates(lev, cell_idx); + amrex::print_state(stateNew, cell_idx); amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; } if (abortOnFofcFailure_ != 0) { From d9ce0909fcf5a5cf8bf0ba6ffd07d8fefc06b4c7 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 13:56:59 +0200 Subject: [PATCH 058/165] remove clangtidy suggestion --- src/RadhydroSimulation.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 673ca16e3..71425fc2b 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -924,8 +924,8 @@ template auto RadhydroSimulation::isCflViolated( template void RadhydroSimulation::printCoordinates(int lev, const amrex::IntVect& cell_idx) { amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real const y_coord = NAN; - amrex::Real const z_coord = NAN; + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; if (AMREX_SPACEDIM > 1) { y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); From b4906b59089aeb726f543022884efb3f6c93e46f Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 15 Sep 2023 21:53:27 +0800 Subject: [PATCH 059/165] drive turb in entire box --- src/PopIII/popiii.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 016331f01..458cd1871 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -291,16 +291,13 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real phi = atan2((y-y0), (x-x0)); - double vx = 0; - double vy = 0; - double vz = 0; + double vx = renorm_amp * dvx(i, j, k); + double vy = renorm_amp * dvy(i, j, k); + double vz = renorm_amp * dvz(i, j, k); if (r <= R_sphere) { state.rho = rhotot; state.T = core_temp; - vx = renorm_amp * dvx(i, j, k); - vy = renorm_amp * dvy(i, j, k); - vz = renorm_amp * dvz(i, j, k); // add rotation to vx and vy vx += (-1.0) * distxy * omega_sphere * std::sin(phi); From e966d79157cdc7f5ab297e87269c88b6aa2f5efe Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 15 Sep 2023 15:09:44 -0400 Subject: [PATCH 060/165] use minmod limiting for PLM --- src/hyperbolic_system.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/hyperbolic_system.hpp b/src/hyperbolic_system.hpp index 4ad5d57a6..2ee15eb7c 100644 --- a/src/hyperbolic_system.hpp +++ b/src/hyperbolic_system.hpp @@ -53,6 +53,11 @@ template class HyperbolicSystem return 0.5 * (sgn(a) + sgn(b)) * std::min(0.5 * std::abs(a + b), std::min(2.0 * std::abs(a), 2.0 * std::abs(b))); } + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static auto minmod(double a, double b) -> double + { + return 0.5 * (sgn(a) + sgn(b)) * std::min(std::abs(a), std::abs(b)); + } + [[nodiscard]] AMREX_GPU_DEVICE AMREX_FORCE_INLINE static auto GetMinmaxSurroundingCell(arrayconst_t &q, int i, int j, int k, int n) -> std::pair; @@ -121,7 +126,7 @@ template void HyperbolicSystem::ReconstructStatesPLM(amrex::MultiFab const &q_mf, amrex::MultiFab &leftState_mf, amrex::MultiFab &rightState_mf, const int nghost, const int nvars) { - // Unlike PPM, PLM with the MC limiter is TVD. + // Unlike PPM, PLM with MC or minmod limiters is TVD. // (There are no spurious oscillations, *except* in the slow-moving shock problem, // which can produce unphysical oscillations even when using upwind Godunov fluxes.) // However, most tests fail when using PLM reconstruction because @@ -151,8 +156,8 @@ void HyperbolicSystem::ReconstructStatesPLM(amrex::MultiFab const &q_ // Use piecewise-linear reconstruction // (This converges at second order in spatial resolution.) - const auto lslope = MC(q(i, j, k, n) - q(i - 1, j, k, n), q(i - 1, j, k, n) - q(i - 2, j, k, n)); - const auto rslope = MC(q(i + 1, j, k, n) - q(i, j, k, n), q(i, j, k, n) - q(i - 1, j, k, n)); + const auto lslope = minmod(q(i, j, k, n) - q(i - 1, j, k, n), q(i - 1, j, k, n) - q(i - 2, j, k, n)); + const auto rslope = minmod(q(i + 1, j, k, n) - q(i, j, k, n), q(i, j, k, n) - q(i - 1, j, k, n)); leftState(i, j, k, n) = q(i - 1, j, k, n) + 0.25 * lslope; // NOLINT rightState(i, j, k, n) = q(i, j, k, n) - 0.25 * rslope; // NOLINT From d8b7da99358f082874902e221d261a7cada96bd4 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 15 Sep 2023 16:48:02 -0400 Subject: [PATCH 061/165] write multifab to disk if hydro update fails --- src/RadhydroSimulation.hpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 42c3f649f..383e735a9 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -892,6 +892,21 @@ void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amre amrex::Print() << "\nQUOKKA FATAL ERROR\n" << "Hydro update exceeded max_retries on level " << lev << ". Cannot continue, crashing...\n" << std::endl; + + // write plotfile or Ascent Blueprint file + amrex::ParallelDescriptor::Barrier(); +#ifdef AMREX_USE_ASCENT + conduit::Node mesh; + amrex::SingleLevelToBlueprint(state_new_cc_[lev], componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); + conduit::Node bpMeshHost; + bpMeshHost.set(mesh); // copy to host mem (needed for Blueprint HDF5 output) + amrex::WriteBlueprintFiles(bpMeshHost, "debug_hydro_state_fatal", istep[lev] + 1, "hdf5"); +#else + WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], time, + istep[lev] + 1); +#endif + amrex::ParallelDescriptor::Barrier(); + amrex::Abort(); } } From 8e70cca081d611b793ddfc9f71b1406ea621936b Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 15 Sep 2023 16:48:41 -0400 Subject: [PATCH 062/165] call MPI_Abort --- src/RadhydroSimulation.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 383e735a9..e5f5bf063 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -907,7 +907,9 @@ void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amre #endif amrex::ParallelDescriptor::Barrier(); - amrex::Abort(); + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } } } From a797259af03a37280f4ecb732b8fba738d626b94 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 18 Sep 2023 23:11:39 +0800 Subject: [PATCH 063/165] debuggin --- src/RadhydroSimulation.hpp | 2 ++ src/hydro_system.hpp | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 2b26a4800..0f0b842de 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -544,6 +544,8 @@ template void RadhydroSimulation::computeAfterEv amrex::Real const Egas0 = initSumCons[RadSystem::gasEnergy_index]; amrex::Real const Egas = state_new_cc_[0].sum(RadSystem::gasEnergy_index) * vol; + amrex::Print() << "after evolve Egas " << Egas << " vol " << vol << std::endl; + amrex::Real Etot0 = NAN; amrex::Real Etot = NAN; if constexpr (Physics_Traits::is_radiation_enabled) { diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 542197c5b..756b2aa33 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -219,6 +219,7 @@ template auto HydroSystem::maxSignalSpeedLocal(a } else { cs = ComputeSoundSpeed(cons[bx], i, j, k); } + return {cs + abs_vel}; }); } @@ -417,7 +418,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::ComputeVelocity template AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::ComputeVelocityX3(amrex::Array4 const &cons, int i, int j, int k) -> amrex::Real -{ +{ amrex::Real const rho = cons(i, j, k, density_index); amrex::Real const vel_z = cons(i, j, k, x3Momentum_index) / rho; return vel_z; @@ -428,8 +429,15 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am { // check if cons(i, j, k) is a valid state const amrex::Real rho = cons(i, j, k, density_index); + const amrex::Real momx = cons(i, j, k, x1Momentum_index); + const amrex::Real momy = cons(i, j, k, x2Momentum_index); + const amrex::Real momz = cons(i, j, k, x3Momentum_index); + const amrex::Real vel = std::pow(std::pow(momx/rho, 2) + std::pow(momy/rho, 2) + std::pow(momz/rho, 2), 0.5); + bool isDensityPositive = (rho > 0.); + bool isVelocityLarge = (vel < 1e7); + bool isMassScalarPositive = true; if constexpr (nmscalars_ > 0) { amrex::GpuArray massScalars_ = RadSystem::ComputeMassScalars(cons, i, j, k); @@ -456,7 +464,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am #endif // return (isDensityPositive && isPressurePositive); - return isDensityPositive && isMassScalarPositive; + return isDensityPositive && isMassScalarPositive && isVelocityLarge; } template From c0e3cd040ffbae7564d81fb4e851949971b3c919 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Tue, 19 Sep 2023 06:35:49 +0800 Subject: [PATCH 064/165] debugging --- src/hydro_system.hpp | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 756b2aa33..84e8328fe 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -289,6 +289,14 @@ template auto HydroSystem::CheckStatesValid(amre thermal_energy, P); return {false}; } + + if (rho != rho || P != P) { + printf("nan state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, + thermal_energy, P); + amrex::Abort("nan here so I abort"); + return {false}; + } + } return {true}; }); @@ -716,6 +724,9 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::ParallelFor(state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { amrex::Real const rho = state[bx](i, j, k, density_index); + if (rho <= 0 || std::isnan(rho)) { + printf("rho is %f", rho); + } amrex::Real const vx1 = state[bx](i, j, k, x1Momentum_index) / rho; amrex::Real const vx2 = state[bx](i, j, k, x2Momentum_index) / rho; amrex::Real const vx3 = state[bx](i, j, k, x3Momentum_index) / rho; @@ -729,14 +740,14 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex if (rho < densityFloor) { rho_new = densityFloor; state[bx](i, j, k, density_index) = rho_new; - state[bx](i, j, k, internalEnergy_index) = Eint * rho_new / rho; + state[bx](i, j, k, internalEnergy_index) = Eint * rho_new / std::abs(rho); state[bx](i, j, k, energy_index) = rho_new * vsq / 2. + (Etot - Ekin); if (nscalars_ > 0) { for (int n = 0; n < nscalars_; ++n) { if (rho_new == 0.0) { state[bx](i, j, k, scalar0_index + n) = 0.0; } else { - state[bx](i, j, k, scalar0_index + n) *= rho / rho_new; + state[bx](i, j, k, scalar0_index + n) *= std::abs(rho) / rho_new; } } } @@ -754,7 +765,7 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex for (int idx = 0; idx < nmscalars_; ++idx) { if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - state[bx](i, j, k, scalar0_index + idx) = small_x * rho; + state[bx](i, j, k, scalar0_index + idx) = small_x * std::abs(rho); } } @@ -766,6 +777,10 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real const Eint_star = Etot - 0.5 * rho_new * vsq; amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); + + if (std::isnan(P_star) || P_star < 0) { + printf("pres is %f", P_star); + } amrex::Real P_new = P_star; if (P_star < pressureFloor) { P_new = pressureFloor; @@ -811,6 +826,11 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); } + + if (std::isnan(state[bx](i, j, k, density_index))) { + printf("rho is nan!!"); + } + }); } From 9e18b885536929d68a2ad6bd06f0bd132a8aa5bb Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 00:06:33 +0800 Subject: [PATCH 065/165] dont do chem if dt < 0 --- src/Chemistry.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index 73582cba0..ed2e79997 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -83,11 +83,19 @@ template void computeChemistry(amrex::MultiFab &mf, const R // call the EOS to set initial internal energy e eos(eos_input_re, chemstate); + if (dt < 0) { + amrex::Abort("Cannot do chemistry with dt < 0!"); + } + // do the actual integration // do it in .cpp so that it is not built at compile time for all tests // which would otherwise slow down compilation due to the large RHS file chemburner(chemstate, dt); + if (std::isnan(chemstate.xn[0]) || std::isnan(chemstate.rho)) { + amrex::Abort("Burner returned NAN"); + } + if (!chemstate.success) { amrex::Abort("VODE integration was unsuccessful!"); } From bb7f5926d64b093b923c2d4820cae461a05763ef Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 00:08:23 +0800 Subject: [PATCH 066/165] use speed ceiling --- src/PopIII/popiii.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 458cd1871..7e021d6a8 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -418,6 +418,7 @@ auto problem_main() -> int sim.doPoissonSolve_ = 1; // enable self-gravity sim.densityFloor_ = 1e-25; sim.tempFloor_ = 2.73 * (30.0 + 1.0); + sim.speedCeiling_ = 3e6; sim.userData_.R_sphere = R_sphere; sim.userData_.numdens_init = numdens_init; From 0668d1a5953b8717367e168d4e415af6e12e89ee Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 19:00:37 +0800 Subject: [PATCH 067/165] ensure pressure eqbm in ICs --- src/PopIII/popiii.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 458cd1871..f445c3bf3 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -265,7 +265,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real const r = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2) + std::pow(z - z0, 2)); amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); - burn_t state; + eos_t state; amrex::Real rhotot = 0.0; for (int n = 0; n < NumSpec; ++n) { @@ -295,9 +295,12 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: double vy = renorm_amp * dvy(i, j, k); double vz = renorm_amp * dvz(i, j, k); + amrex::Real pres = 0.0; + if (r <= R_sphere) { state.rho = rhotot; state.T = core_temp; + eos(eos_input_rt, state); // add rotation to vx and vy vx += (-1.0) * distxy * omega_sphere * std::sin(phi); @@ -305,11 +308,12 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: } else { state.rho = 0.01 * rhotot; - state.T = 1e2 * core_temp; // can use EOS.hpp function to solve for temp + state.p = 3.595730e-10; //pressure equilibrium - this is the pressure within the core + eos(eos_input_rp, state); + } // call the EOS to set initial internal energy e - eos(eos_input_rt, state); amrex::Real e = state.rho * state.e; //amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; @@ -355,7 +359,7 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); - if (l_Jeans < (N_cells * dx) && rho > 2e-20) { + if (l_Jeans < (N_cells * dx) && rho > 5e-20) { tag(i, j, k) = amrex::TagBox::SET; } From d15dc337627f253cbe0a803df37cfd5703d770cc Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 19:02:20 +0800 Subject: [PATCH 068/165] checks --- src/RadhydroSimulation.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 0f0b842de..5102c2ad3 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -708,9 +708,12 @@ template void RadhydroSimulation::applyPoissonGr template void RadhydroSimulation::FixupState(int lev) { BL_PROFILE("RadhydroSimulation::FixupState()"); + amrex::ParallelDescriptor::Barrier(); // fix hydro state + amrex::Print() << "enforce limits state_new_cc " << std::endl; HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); + amrex::Print() << "enforce limits state_old_cc " << std::endl; HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); // sync internal energy and total energy HydroSystem::SyncDualEnergy(state_new_cc_[lev]); @@ -1104,6 +1107,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum + amrex::Print() << "enforce limits fofc 1 " << std::endl; HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); if (useDualEnergy_ == 1) { @@ -1187,6 +1191,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum + amrex::Print() << "enforce limits prevent vacuum stage 2 " << std::endl; HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); if (useDualEnergy_ == 1) { From 4898868976f5712ff3b6d7675e08936f32bf61ae Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 20:04:36 +0800 Subject: [PATCH 069/165] add velx and pressure as derived vars --- src/PopIII/popiii.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 1a965621a..a9f0e2196 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -387,6 +387,34 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str output[bx](i, j, k, ncomp) = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); }); } + + if (dname == "pressure") { + + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + amrex::Real Pgas = HydroSystem::ComputePressure(state[bx], i, j, k); + output[bx](i, j, k, ncomp) = Pgas; + }); + } + + + if (dname == "velx") { + + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + Real const xmom = state[bx](i, j, k, HydroSystem::x1Momentum_index); + output[bx](i, j, k, ncomp) = xmom / rho; + }); + } + } auto problem_main() -> int From 78bdc4c4baa6da89ce5b5fd202ae0035c7e9850e Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 20:45:30 +0800 Subject: [PATCH 070/165] add exmaple of refinement restricted to certain cell positions to avoid refinement at cloud edge --- src/PopIII/popiii.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index a9f0e2196..20276f91f 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -343,6 +343,14 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra const amrex::Real G = Gconst_; const amrex::Real dx = geom[lev].CellSizeArray()[0]; + auto const &prob_lo = geom[lev].ProbLoArray(); + auto const &prob_hi = geom[lev].ProbHiArray(); + + amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); + amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); + amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); + + for (amrex::MFIter mfi(state_new_cc_[lev]); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.validbox(); const auto state = state_new_cc_[lev].const_array(mfi); @@ -351,6 +359,10 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real const x = prob_lo[0] + (i + static_cast(0.5)) * dx; + amrex::Real const y = prob_lo[1] + (j + static_cast(0.5)) * dx; + amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx; + Real const rho = state(i, j, k, nidx); Real pressure = HydroSystem::ComputePressure(state, i, j, k); amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state, i, j, k); @@ -359,7 +371,7 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); - if (l_Jeans < (N_cells * dx) && rho > 5e-20) { + if (l_Jeans < (N_cells * dx) && rho > 5e-20 && x > -6.172e17 && x < 1.543e18 && y > -6.172e17) { tag(i, j, k) = amrex::TagBox::SET; } From 04aea4f92f0d33231c59c366d8f673246390340c Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 21:32:57 +0800 Subject: [PATCH 071/165] abort if rho = 0; --- src/hydro_system.hpp | 215 ++++++++++++++++++++++++------------------- 1 file changed, 118 insertions(+), 97 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 84e8328fe..b83927912 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -275,8 +275,8 @@ template auto HydroSystem::CheckStatesValid(amre const auto thermal_energy = E - kinetic_energy; const auto P = ComputePressure(cons[bx], i, j, k); - bool negativeDensity = (rho <= 0.); - bool negativePressure = (P <= 0.); + bool negativeDensity = (rho < 0.); + bool negativePressure = (P < 0.); if constexpr (is_eos_isothermal()) { if (negativeDensity) { @@ -284,16 +284,20 @@ template auto HydroSystem::CheckStatesValid(amre return {false}; } } else { - if (negativeDensity || negativePressure) { - printf("invalid state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, - thermal_energy, P); - return {false}; - } + //if (negativeDensity || negativePressure) { + // printf("invalid state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, + // thermal_energy, P); + // return {false}; + //} + - if (rho != rho || P != P) { + + if (std::isnan(rho) || std::isnan(P)) { printf("nan state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, thermal_energy, P); - amrex::Abort("nan here so I abort"); + return {false}; + } else if (rho == 0.0 || P == 0.0) { + printf("zero state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, thermal_energy, P); return {false}; } @@ -724,111 +728,128 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::ParallelFor(state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { amrex::Real const rho = state[bx](i, j, k, density_index); - if (rho <= 0 || std::isnan(rho)) { - printf("rho is %f", rho); - } - amrex::Real const vx1 = state[bx](i, j, k, x1Momentum_index) / rho; - amrex::Real const vx2 = state[bx](i, j, k, x2Momentum_index) / rho; - amrex::Real const vx3 = state[bx](i, j, k, x3Momentum_index) / rho; - amrex::Real const vsq = (vx1 * vx1 + vx2 * vx2 + vx3 * vx3); - amrex::Real const v_abs = std::sqrt(vx1 * vx1 + vx2 * vx2 + vx3 * vx3); - amrex::Real Etot = state[bx](i, j, k, energy_index); - amrex::Real Eint = state[bx](i, j, k, internalEnergy_index); - amrex::Real Ekin = rho * vsq / 2.; - amrex::Real rho_new = rho; - - if (rho < densityFloor) { - rho_new = densityFloor; - state[bx](i, j, k, density_index) = rho_new; - state[bx](i, j, k, internalEnergy_index) = Eint * rho_new / std::abs(rho); - state[bx](i, j, k, energy_index) = rho_new * vsq / 2. + (Etot - Ekin); - if (nscalars_ > 0) { - for (int n = 0; n < nscalars_; ++n) { - if (rho_new == 0.0) { - state[bx](i, j, k, scalar0_index + n) = 0.0; - } else { - state[bx](i, j, k, scalar0_index + n) *= std::abs(rho) / rho_new; + + if (rho != 0.0 && !std::isnan(rho)) { + amrex::Real const vx1 = state[bx](i, j, k, x1Momentum_index) / rho; + amrex::Real const vx2 = state[bx](i, j, k, x2Momentum_index) / rho; + amrex::Real const vx3 = state[bx](i, j, k, x3Momentum_index) / rho; + amrex::Real const vsq = (vx1 * vx1 + vx2 * vx2 + vx3 * vx3); + amrex::Real const v_abs = std::sqrt(vx1 * vx1 + vx2 * vx2 + vx3 * vx3); + amrex::Real Etot = state[bx](i, j, k, energy_index); + amrex::Real Eint = state[bx](i, j, k, internalEnergy_index); + amrex::Real Ekin = rho * vsq / 2.; + amrex::Real rho_new = rho; + + if (std::isnan(rho)) { + printf("rho = %e Eint = %e Ekin = %e v_abs = %e \n", rho, Eint, Ekin, v_abs); + if (nmscalars_ > 0) { + + for (int idx = 0; idx < nmscalars_; ++idx) { + if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { + printf("nm is %e \n", state[bx](i, j, k, scalar0_index + idx)); + } + } + } + + } + + if (rho < densityFloor) { + rho_new = densityFloor; + state[bx](i, j, k, density_index) = rho_new; + state[bx](i, j, k, internalEnergy_index) = Eint * rho_new / rho; + state[bx](i, j, k, energy_index) = rho_new * vsq / 2. + (Etot - Ekin); + if (nscalars_ > 0) { + for (int n = 0; n < nscalars_; ++n) { + if (rho_new == 0.0) { + state[bx](i, j, k, scalar0_index + n) = 0.0; + } else { + state[bx](i, j, k, scalar0_index + n) *= rho / rho_new; + } } } } - } - if (v_abs > speedCeiling) { - amrex::Real rescale_factor = speedCeiling / v_abs; - state[bx](i, j, k, x1Momentum_index) *= rescale_factor; - state[bx](i, j, k, x2Momentum_index) *= rescale_factor; - state[bx](i, j, k, x3Momentum_index) *= rescale_factor; - } + if (v_abs > speedCeiling) { + amrex::Real rescale_factor = speedCeiling / v_abs; + state[bx](i, j, k, x1Momentum_index) *= rescale_factor; + state[bx](i, j, k, x2Momentum_index) *= rescale_factor; + state[bx](i, j, k, x3Momentum_index) *= rescale_factor; + } - // Enforcing Limits on mass scalars - if (nmscalars_ > 0) { + // Enforcing Limits on mass scalars + if (nmscalars_ > 0) { - for (int idx = 0; idx < nmscalars_; ++idx) { - if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - state[bx](i, j, k, scalar0_index + idx) = small_x * std::abs(rho); - } - } + for (int idx = 0; idx < nmscalars_; ++idx) { + if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { + state[bx](i, j, k, scalar0_index + idx) = small_x * rho; + } + } - } + } - // Enforcing Limits on temperature estimated from Etot and Ekin - if (!HydroSystem::is_eos_isothermal()) { - // recompute gas energy (to prevent P < 0) - amrex::Real const Eint_star = Etot - 0.5 * rho_new * vsq; - amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); - amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); + // Enforcing Limits on temperature estimated from Etot and Ekin + if (!HydroSystem::is_eos_isothermal()) { + // recompute gas energy (to prevent P < 0) + amrex::Real const Eint_star = Etot - 0.5 * rho_new * vsq; + amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); - if (std::isnan(P_star) || P_star < 0) { - printf("pres is %f", P_star); - } - amrex::Real P_new = P_star; - if (P_star < pressureFloor) { - P_new = pressureFloor; + if (std::isnan(P_star) || P_star < 0) { + printf("pres is %f", P_star); + } + amrex::Real P_new = P_star; + if (P_star < pressureFloor) { + P_new = pressureFloor; #pragma nv_diag_suppress divide_by_zero - amrex::Real const Etot_new = quokka::EOS::ComputeEintFromPres(rho_new, P_new, massScalars) + 0.5 * rho_new * vsq; - state[bx](i, j, k, HydroSystem::energy_index) = Etot_new; + amrex::Real const Etot_new = quokka::EOS::ComputeEintFromPres(rho_new, P_new, massScalars) + 0.5 * rho_new * vsq; + state[bx](i, j, k, HydroSystem::energy_index) = Etot_new; + } } - } - // re-obtain Ekin and Etot for putting limits on Temperature - if (rho_new == 0.0) { - Ekin = Etot = 0.0; - } else { - Ekin = std::pow(state[bx](i, j, k, x1Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; - Ekin += std::pow(state[bx](i, j, k, x2Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; - Ekin += std::pow(state[bx](i, j, k, x3Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; - Etot = state[bx](i, j, k, energy_index); - } - amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); - amrex::Real primTemp = quokka::EOS::ComputeTgasFromEint(rho, (Etot - Ekin), massScalars); + // re-obtain Ekin and Etot for putting limits on Temperature + if (rho_new == 0.0) { + Ekin = Etot = 0.0; + } else { + Ekin = std::pow(state[bx](i, j, k, x1Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; + Ekin += std::pow(state[bx](i, j, k, x2Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; + Ekin += std::pow(state[bx](i, j, k, x3Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; + Etot = state[bx](i, j, k, energy_index); + } + amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + amrex::Real primTemp = quokka::EOS::ComputeTgasFromEint(rho, (Etot - Ekin), massScalars); - if (primTemp > tempCeiling) { - amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); - state[bx](i, j, k, energy_index) = Ekin + prim_eint; - } + if (primTemp > tempCeiling) { + amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); + state[bx](i, j, k, energy_index) = Ekin + prim_eint; + } - if (primTemp < tempFloor) { - amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); - state[bx](i, j, k, energy_index) = Ekin + prim_eint; - } + if (primTemp < tempFloor) { + amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); + state[bx](i, j, k, energy_index) = Ekin + prim_eint; + } - // Enforcing Limits on Auxiliary temperature estimated from Eint - Eint = state[bx](i, j, k, internalEnergy_index); - amrex::Real auxTemp = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); + // Enforcing Limits on Auxiliary temperature estimated from Eint + Eint = state[bx](i, j, k, internalEnergy_index); + amrex::Real auxTemp = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); - if (auxTemp > tempCeiling) { - state[bx](i, j, k, internalEnergy_index) = - quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); - state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); - } + if (auxTemp > tempCeiling) { + state[bx](i, j, k, internalEnergy_index) = + quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); + state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); + } - if (auxTemp < tempFloor) { - state[bx](i, j, k, internalEnergy_index) = - quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); - state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); - } + if (auxTemp < tempFloor) { + state[bx](i, j, k, internalEnergy_index) = + quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); + state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); + } - if (std::isnan(state[bx](i, j, k, density_index))) { - printf("rho is nan!!"); + if (std::isnan(state[bx](i, j, k, density_index)) || state[bx](i, j, k, density_index) == 0.0) { + printf("rho out is nan or zero even though rho in is not nan %e !! ", state[bx](i, j, k, density_index)); + } + } else if (std::isnan(rho)) { + printf("rho in is nan howcome %e !", rho); + } else { + printf("rho in is zero woah %e !", rho); } }); From 6691333976c34c97037aee248ba1abb41975457a Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 23:21:26 +0800 Subject: [PATCH 072/165] check hydro state in more locations --- src/RadhydroSimulation.hpp | 39 ++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 5102c2ad3..5bf548fa7 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -433,11 +433,11 @@ template auto RadhydroSimulation::computeExtraPh return std::numeric_limits::max(); } -#if !defined(NDEBUG) +//#if !defined(NDEBUG) #define CHECK_HYDRO_STATES(mf) checkHydroStates(mf, __FILE__, __LINE__) -#else -#define CHECK_HYDRO_STATES(mf) -#endif +//#else +//#define CHECK_HYDRO_STATES(mf) +//#endif template void RadhydroSimulation::checkHydroStates(amrex::MultiFab &mf, char const *file, int line) { @@ -451,6 +451,7 @@ template void RadhydroSimulation::checkHydroStat amrex::Print() << "Writing checkpoint for debugging...\n"; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); + amrex::Abort("Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")"); } } @@ -999,6 +1000,16 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // update ghost zones [old timestep] fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); + bool validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); + if (!validStates) { + amrex::Print() << "Hydro states invalid after fillBC in RK \n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid after fillBC in RK "); + } + + // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) if (lowLevelDebuggingOutput_ == 1) { #ifdef AMREX_USE_ASCENT @@ -1019,6 +1030,16 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); + validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); + if (!validStates) { + amrex::Print() << "Hydro states invalid before RK2 stage 1 \n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid before RK2 stage 1 "); + } + + // Stage 1 of RK2-SSP { // advance all grids on local processor (Stage 1 of integrator) @@ -1122,6 +1143,16 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } amrex::Gpu::streamSynchronizeAll(); + validStates = HydroSystem::CheckStatesValid(state_inter_cc_); + if (!validStates) { + amrex::Print() << "Hydro states invalid after RK2 stage 1 \n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid after RK2 stage "); + } + + // Stage 2 of RK2-SSP if (integratorOrder_ == 2) { // update ghost zones [intermediate stage stored in state_inter_cc_] From 73eecb5b041fad081e3d0bb6717a6b2825f85f27 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 23:24:48 +0800 Subject: [PATCH 073/165] fix tab --- src/RadhydroSimulation.hpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 5bf548fa7..ba920797f 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1000,14 +1000,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // update ghost zones [old timestep] fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); - bool validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); - if (!validStates) { - amrex::Print() << "Hydro states invalid after fillBC in RK \n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid after fillBC in RK "); - } + bool validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); + if (!validStates) { + amrex::Print() << "Hydro states invalid after fillBC in RK \n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid after fillBC in RK "); + } // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) @@ -1030,14 +1030,14 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); - validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); - if (!validStates) { - amrex::Print() << "Hydro states invalid before RK2 stage 1 \n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid before RK2 stage 1 "); - } + validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); + if (!validStates) { + amrex::Print() << "Hydro states invalid before RK2 stage 1 \n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid before RK2 stage 1 "); + } // Stage 1 of RK2-SSP From 0d4f67381dcc822c698df1888f08ac12efd9821d Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 20 Sep 2023 17:26:47 +0200 Subject: [PATCH 074/165] reduce validstates boolean --- src/RadhydroSimulation.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index ba920797f..81369238c 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1001,6 +1001,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); bool validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); + amrex::ParallelDescriptor::ReduceBoolAnd(validStates); if (!validStates) { amrex::Print() << "Hydro states invalid after fillBC in RK \n"; amrex::Print() << "Writing checkpoint for debugging...\n"; @@ -1031,6 +1032,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); + amrex::ParallelDescriptor::ReduceBoolAnd(validStates); + if (!validStates) { amrex::Print() << "Hydro states invalid before RK2 stage 1 \n"; amrex::Print() << "Writing checkpoint for debugging...\n"; @@ -1144,6 +1147,8 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Gpu::streamSynchronizeAll(); validStates = HydroSystem::CheckStatesValid(state_inter_cc_); + amrex::ParallelDescriptor::ReduceBoolAnd(validStates); + if (!validStates) { amrex::Print() << "Hydro states invalid after RK2 stage 1 \n"; amrex::Print() << "Writing checkpoint for debugging...\n"; From d8f405d7bdf267e07357d8075c8b0197cf67ba04 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 04:10:51 +0800 Subject: [PATCH 075/165] add function call name keyword --- src/RadhydroSimulation.hpp | 8 ++++---- src/hydro_system.hpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 81369238c..997a22224 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -713,9 +713,9 @@ template void RadhydroSimulation::FixupState(int // fix hydro state amrex::Print() << "enforce limits state_new_cc " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev], "fixup_new"); amrex::Print() << "enforce limits state_old_cc " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev], "fixup_old"); // sync internal energy and total energy HydroSystem::SyncDualEnergy(state_new_cc_[lev]); } @@ -1132,7 +1132,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // prevent vacuum amrex::Print() << "enforce limits fofc 1 " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew, "RK2-stage-1"); if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) @@ -1228,7 +1228,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // prevent vacuum amrex::Print() << "enforce limits prevent vacuum stage 2 " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal, "RK2-stage-2"); if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index b83927912..c42223b0e 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -109,7 +109,7 @@ template class HydroSystem : public HyperbolicSystem posvec) -> amrex::GpuArray; static void EnforceLimits(amrex::Real densityFloor, amrex::Real pressureFloor, amrex::Real speedCeiling, amrex::Real tempCeiling, - amrex::Real const tempFloor, amrex::MultiFab &state_mf); + amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString); static void AddInternalEnergyPdV(amrex::MultiFab &rhs_mf, amrex::MultiFab const &consVar_mf, amrex::GpuArray dx, std::array const &faceVelArray, amrex::iMultiFab const &redoFlag_mf); @@ -721,7 +721,7 @@ void HydroSystem::FlattenShocks(amrex::MultiFab const &q_mf, amrex::M // floors and ceilings which can be set in the param file template void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex::Real const pressureFloor, amrex::Real const speedCeiling, - amrex::Real const tempCeiling, amrex::Real const tempFloor, amrex::MultiFab &state_mf) + amrex::Real const tempCeiling, amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString) { auto state = state_mf.arrays(); @@ -847,9 +847,9 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex printf("rho out is nan or zero even though rho in is not nan %e !! ", state[bx](i, j, k, density_index)); } } else if (std::isnan(rho)) { - printf("rho in is nan howcome %e !", rho); + printf("rho in is nan howcome %e called by %s! \n", rho, inputString); } else { - printf("rho in is zero woah %e !", rho); + printf("rho in is zero woah %e called by %s ! \n", rho, inputString); } }); From 5eae125be7197b50ecd41ebcad6d4e9c7a3df2c8 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 16:26:10 +0800 Subject: [PATCH 076/165] enforceLimits returns a bool --- src/hydro_system.hpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index c42223b0e..21009031f 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -108,8 +108,8 @@ template class HydroSystem : public HyperbolicSystem posvec) -> amrex::GpuArray; - static void EnforceLimits(amrex::Real densityFloor, amrex::Real pressureFloor, amrex::Real speedCeiling, amrex::Real tempCeiling, - amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString); + static auto EnforceLimits(amrex::Real densityFloor, amrex::Real pressureFloor, amrex::Real speedCeiling, amrex::Real tempCeiling, + amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString) -> bool; static void AddInternalEnergyPdV(amrex::MultiFab &rhs_mf, amrex::MultiFab const &consVar_mf, amrex::GpuArray dx, std::array const &faceVelArray, amrex::iMultiFab const &redoFlag_mf); @@ -720,13 +720,14 @@ void HydroSystem::FlattenShocks(amrex::MultiFab const &q_mf, amrex::M // to ensure that physical quantities are within reasonable // floors and ceilings which can be set in the param file template -void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex::Real const pressureFloor, amrex::Real const speedCeiling, - amrex::Real const tempCeiling, amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString) +auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex::Real const pressureFloor, amrex::Real const speedCeiling, + amrex::Real const tempCeiling, amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString) -> bool { auto state = state_mf.arrays(); + amrex::Real strcall = 0.0; - amrex::ParallelFor(state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + return amrex::ParReduce(amrex::TypeList{}, amrex::TypeList{}, state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept -> amrex::GpuTuple { amrex::Real const rho = state[bx](i, j, k, density_index); if (rho != 0.0 && !std::isnan(rho)) { @@ -846,13 +847,22 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex if (std::isnan(state[bx](i, j, k, density_index)) || state[bx](i, j, k, density_index) == 0.0) { printf("rho out is nan or zero even though rho in is not nan %e !! ", state[bx](i, j, k, density_index)); } + + return {true}; } else if (std::isnan(rho)) { - printf("rho in is nan howcome %e called by %s! \n", rho, inputString); + //printf("rho in is nan howcome %e! \n", rho); + //strcall = 1.0; + return {false}; } else { - printf("rho in is zero woah %e called by %s ! \n", rho, inputString); + //printf("rho in is zero woah %e! \n", rho); + //strcall = 1.0; + return {false}; } }); + + amrex::Print() << "called by " << inputString << std::endl; + } template From 19c5486b9dbbb237e0b81f58caedb6f30a19ddbe Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 16:26:36 +0800 Subject: [PATCH 077/165] enforceLimits returns a bool --- src/RadhydroSimulation.hpp | 42 ++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 997a22224..e2d564711 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -713,9 +713,26 @@ template void RadhydroSimulation::FixupState(int // fix hydro state amrex::Print() << "enforce limits state_new_cc " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev], "fixup_new"); + bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev], "fixup_new"); + amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); + if (!enforchk) { + amrex::Print() << "enforce limits has rho = 0 fixup state_new_cc " << std::endl; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort(); + } + amrex::Print() << "enforce limits state_old_cc " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev], "fixup_old"); + enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev], "fixup_old"); + amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); + if (!enforchk) { + amrex::Print() << "enforce limits has rho = 0 fixup state_old_cc " << std::endl; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort(); + } + + // sync internal energy and total energy HydroSystem::SyncDualEnergy(state_new_cc_[lev]); } @@ -1132,7 +1149,16 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // prevent vacuum amrex::Print() << "enforce limits fofc 1 " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew, "RK2-stage-1"); + bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew, "RK2-stage-1"); + amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); + if (!enforchk) { + amrex::Print() << "enforce limits has rho = 0 fixup fofc 1 stage 1 rk2 " << std::endl; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort(); + } + + if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) @@ -1228,7 +1254,15 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // prevent vacuum amrex::Print() << "enforce limits prevent vacuum stage 2 " << std::endl; - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal, "RK2-stage-2"); + bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal, "RK2-stage-2"); + amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); + if (!enforchk) { + amrex::Print() << "enforce limits has rho = 0 fixup prevent vacuum stage 2 rk2 " << std::endl; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort(); + } + if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) From 57386d82a2267b769f2c76ef46a5d857adc86861 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 17:31:39 +0800 Subject: [PATCH 078/165] write pltfile when rho = 0 encountered and abort --- src/RadhydroSimulation.hpp | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index e2d564711..eb8c6561b 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -719,7 +719,12 @@ template void RadhydroSimulation::FixupState(int amrex::Print() << "enforce limits has rho = 0 fixup state_new_cc " << std::endl; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); - amrex::Abort(); + amrex::ParallelDescriptor::Barrier(); + amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], 0, istep[lev] + 1); + amrex::ParallelDescriptor::Barrier(); + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } } amrex::Print() << "enforce limits state_old_cc " << std::endl; @@ -729,7 +734,12 @@ template void RadhydroSimulation::FixupState(int amrex::Print() << "enforce limits has rho = 0 fixup state_old_cc " << std::endl; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); - amrex::Abort(); + amrex::ParallelDescriptor::Barrier(); + amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_old_cc_[lev], componentNames_cc_, geom[lev], 0, istep[lev] + 1); + amrex::ParallelDescriptor::Barrier(); + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } } @@ -1155,7 +1165,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "enforce limits has rho = 0 fixup fofc 1 stage 1 rk2 " << std::endl; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); - amrex::Abort(); + amrex::ParallelDescriptor::Barrier(); + amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateNew, componentNames_cc_, geom[lev], 0, istep[lev] + 1); + amrex::ParallelDescriptor::Barrier(); + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } } @@ -1260,7 +1275,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "enforce limits has rho = 0 fixup prevent vacuum stage 2 rk2 " << std::endl; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); - amrex::Abort(); + amrex::ParallelDescriptor::Barrier(); + amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateFinal, componentNames_cc_, geom[lev], 0, istep[lev] + 1); + amrex::ParallelDescriptor::Barrier(); + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } } From 62db47e5105ed1421de2a93e86a56455845dd301 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 11:32:53 +0200 Subject: [PATCH 079/165] indent --- src/RadhydroSimulation.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index eb8c6561b..1cb935a8f 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1275,12 +1275,12 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::Print() << "enforce limits has rho = 0 fixup prevent vacuum stage 2 rk2 " << std::endl; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); - amrex::ParallelDescriptor::Barrier(); - amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateFinal, componentNames_cc_, geom[lev], 0, istep[lev] + 1); - amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Barrier(); + amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateFinal, componentNames_cc_, geom[lev], 0, istep[lev] + 1); + amrex::ParallelDescriptor::Barrier(); + if (amrex::ParallelDescriptor::IOProcessor()) { amrex::ParallelDescriptor::Abort(); - } + } } From f1e2c294a23a1e66666f451fe18086b93cfa3e36 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 22:09:16 +0800 Subject: [PATCH 080/165] added dummy test --- src/CopIII/CMakeLists.txt | 75 ++++++ src/CopIII/ascent_actions.yaml | 30 +++ src/CopIII/copiii.cpp | 469 +++++++++++++++++++++++++++++++++ src/CopIII/copiii.hpp | 23 ++ 4 files changed, 597 insertions(+) create mode 100644 src/CopIII/CMakeLists.txt create mode 100644 src/CopIII/ascent_actions.yaml create mode 100644 src/CopIII/copiii.cpp create mode 100644 src/CopIII/copiii.hpp diff --git a/src/CopIII/CMakeLists.txt b/src/CopIII/CMakeLists.txt new file mode 100644 index 000000000..9205bc8d6 --- /dev/null +++ b/src/CopIII/CMakeLists.txt @@ -0,0 +1,75 @@ +if (AMReX_SPACEDIM EQUAL 3) + # Define a custom target that runs the Python script to produce the input perturbations file + + + set(microphysics_network_name "primordial_chem") #this will override network_name to primordial_chem for this directory only + setup_target_for_microphysics_compilation(${microphysics_network_name} "${CMAKE_CURRENT_BINARY_DIR}/") + + #use the BEFORE keyword so that these files get priority in compilation for targets in this directory + #this is critical to ensure the correct Microphysics files are linked to primordial chem target + include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H") + + add_executable(copiii copiii.cpp ../TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) + target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM + + if(AMReX_GPU_BACKEND MATCHES "CUDA") + setup_target_for_cuda_compilation(popiii) + endif() + + #need h5py to run perturbations.py file below + execute_process( + COMMAND Python3::Interpreter -c "h5py" + RESULT_VARIABLE EXIT_CODE + OUTPUT_QUIET + ) + + #re-run cmake if this file is changed + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/CopIII.in) + + # Read the 'PopIII.in' file line by line + file(READ ${CMAKE_SOURCE_DIR}/tests/CopIII.in pop_in_contents) + + # Split the file contents into lines + string(REPLACE "\n" ";" pop_in_lines "${pop_in_contents}") + + # Iterate through the lines and look for lines starting with 'amr.n_cell' + foreach(line IN LISTS pop_in_lines) + if (line MATCHES "^amr\\.n_cell.*") + set(match_line "${line}") + break() + endif() + endforeach() + + if (match_line) + message(STATUS "picked line is ${match_line}") + # Use a regular expression to find consecutive digit + string(REGEX MATCHALL "[0-9]+" digits_list "${match_line}") + # Iterate through the matched digits and extract the first one + list(GET digits_list 0 first_digits) + # Check if matched digits were found + if (first_digits) + #first_digits give the box size, but you also want box size / 2 as kmax in the script below + math(EXPR int_first_digits "${first_digits}") + # Check if the conversion was successful + if (int_first_digits) + # Divide the integer by two + math(EXPR result "${int_first_digits} / 2") + else() + message(FATAL_ERROR "Conversion to integer failed, so cannot find kmax!") + endif() + message(STATUS "Will create initial perturbations of box size ${first_digits} and kmax ${result}") + + else() + message(FATAL_ERROR "No box size found to create initial perturbations for!") + endif() + else() + message(FATAL_ERROR "No matching line found in ${CMAKE_SOURCE_DIR}/tests/CopIII.in!") + endif() + + + add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=1.8 --f_solenoidal=0.66667 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + add_test(NAME CopIII COMMAND copiii CopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP CopIII_fixture) + set_tests_properties(CopIII PROPERTIES FIXTURES_REQUIRED CopIII_fixture) + +endif() diff --git a/src/CopIII/ascent_actions.yaml b/src/CopIII/ascent_actions.yaml new file mode 100644 index 000000000..6515c4b84 --- /dev/null +++ b/src/CopIII/ascent_actions.yaml @@ -0,0 +1,30 @@ +- + action: "add_pipelines" + pipelines: + pl1: + f2: + type: "slice" + params: + point: + x: 0. + y: 0. + z: 0. + normal: + x: 0.0 + y: 0.0 + z: 1.0 +- + action: "add_scenes" + scenes: + s1: + plots: + p1: + type: "pseudocolor" + field: "log_density" + pipeline: "pl1" + renders: + r1: + image_prefix: "dens%05d" + annotations: "true" + camera: + zoom: 1.5 diff --git a/src/CopIII/copiii.cpp b/src/CopIII/copiii.cpp new file mode 100644 index 000000000..8cc05f54d --- /dev/null +++ b/src/CopIII/copiii.cpp @@ -0,0 +1,469 @@ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file copiii.cpp +/// \brief Defines a test problem for Pop III star formation. +/// Author: Piyush Sharda (Leiden University, 2023) +/// +#include +#include +#include + +#include "AMReX.H" +#include "AMReX_Arena.H" +#include "AMReX_BC_TYPES.H" +#include "AMReX_BLassert.H" +#include "AMReX_Config.H" +#include "AMReX_FabArrayUtility.H" +#include "AMReX_MultiFab.H" +#include "AMReX_ParallelDescriptor.H" +#include "AMReX_ParmParse.H" +#include "AMReX_Print.H" +#include "AMReX_SPACE.H" +#include "AMReX_TableData.H" + +#include "RadhydroSimulation.hpp" +#include "SimulationData.hpp" +#include "TurbDataReader.hpp" +#include "hydro_system.hpp" +#include "copiii.hpp" + +#include "actual_eos_data.H" +#include "burn_type.H" +#include "eos.H" +#include "extern_parameters.H" +#include "network.H" + +using amrex::Real; + +struct CopIII { +}; + +template <> struct HydroSystem_Traits { + static constexpr bool reconstruct_eint = true; +}; + +template <> struct Physics_Traits { + // cell-centred + static constexpr bool is_hydro_enabled = true; + static constexpr int numMassScalars = NumSpec; // number of chemical species + static constexpr int numPassiveScalars = numMassScalars + 0; // we only have mass scalars + static constexpr bool is_radiation_enabled = false; + // face-centred + static constexpr bool is_mhd_enabled = false; +}; + + +template <> struct SimulationData { + // real-space perturbation fields + amrex::TableData dvx; + amrex::TableData dvy; + amrex::TableData dvz; + amrex::Real dv_rms_generated{}; + amrex::Real dv_rms_target{}; + amrex::Real rescale_factor{}; + + // cloud parameters + amrex::Real R_sphere{}; + amrex::Real numdens_init{}; + amrex::Real omega_sphere{}; + + amrex::Real small_temp{}; + amrex::Real small_dens{}; + amrex::Real temperature{}; + amrex::Real primary_species_1{}; + amrex::Real primary_species_2{}; + amrex::Real primary_species_3{}; + amrex::Real primary_species_4{}; + amrex::Real primary_species_5{}; + amrex::Real primary_species_6{}; + amrex::Real primary_species_7{}; + amrex::Real primary_species_8{}; + amrex::Real primary_species_9{}; + amrex::Real primary_species_10{}; + amrex::Real primary_species_11{}; + amrex::Real primary_species_12{}; + amrex::Real primary_species_13{}; + amrex::Real primary_species_14{}; + + +}; + +template <> void RadhydroSimulation::preCalculateInitialConditions() +{ + + // initialize microphysics routines + init_extern_parameters(); + + // parmparse species and temperature + amrex::ParmParse pp("primordial_chem"); + userData_.small_temp = 1e1; + pp.query("small_temp", userData_.small_temp); + + userData_.small_dens = 1e-60; + pp.query("small_dens", userData_.small_dens); + + userData_.temperature = 1e1; + pp.query("temperature", userData_.temperature); + + userData_.primary_species_1 = 1.0e0_rt; + userData_.primary_species_2 = 0.0e0_rt; + userData_.primary_species_3 = 0.0e0_rt; + userData_.primary_species_4 = 0.0e0_rt; + userData_.primary_species_5 = 0.0e0_rt; + userData_.primary_species_6 = 0.0e0_rt; + userData_.primary_species_7 = 0.0e0_rt; + userData_.primary_species_8 = 0.0e0_rt; + userData_.primary_species_9 = 0.0e0_rt; + userData_.primary_species_10 = 0.0e0_rt; + userData_.primary_species_11 = 0.0e0_rt; + userData_.primary_species_12 = 0.0e0_rt; + userData_.primary_species_13 = 0.0e0_rt; + userData_.primary_species_14 = 0.0e0_rt; + + pp.query("primary_species_1", userData_.primary_species_1); + pp.query("primary_species_2", userData_.primary_species_2); + pp.query("primary_species_3", userData_.primary_species_3); + pp.query("primary_species_4", userData_.primary_species_4); + pp.query("primary_species_5", userData_.primary_species_5); + pp.query("primary_species_6", userData_.primary_species_6); + pp.query("primary_species_7", userData_.primary_species_7); + pp.query("primary_species_8", userData_.primary_species_8); + pp.query("primary_species_9", userData_.primary_species_9); + pp.query("primary_species_10", userData_.primary_species_10); + pp.query("primary_species_11", userData_.primary_species_11); + pp.query("primary_species_12", userData_.primary_species_12); + pp.query("primary_species_13", userData_.primary_species_13); + pp.query("primary_species_14", userData_.primary_species_14); + + eos_init(userData_.small_temp, userData_.small_dens); + network_init(); + + + static bool isSamplingDone = false; + if (!isSamplingDone) { + // read perturbations from file + turb_data turbData; + amrex::ParmParse pp("perturb"); + std::string turbdata_filename; + pp.query("filename", turbdata_filename); + initialize_turbdata(turbData, turbdata_filename); + + // copy to pinned memory + auto pinned_dvx = get_tabledata(turbData.dvx); + auto pinned_dvy = get_tabledata(turbData.dvy); + auto pinned_dvz = get_tabledata(turbData.dvz); + + // compute normalisation + userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); + amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; + + const Real rms_dv_target = 1.8050e5; + const Real rms_dv_actual = userData_.dv_rms_generated; + userData_.rescale_factor = rms_dv_target / rms_dv_actual; + + // copy to GPU + userData_.dvx.resize(pinned_dvx.lo(), pinned_dvx.hi()); + userData_.dvx.copy(pinned_dvx); + + userData_.dvy.resize(pinned_dvy.lo(), pinned_dvy.hi()); + userData_.dvy.copy(pinned_dvy); + + userData_.dvz.resize(pinned_dvz.lo(), pinned_dvz.hi()); + userData_.dvz.copy(pinned_dvz); + + isSamplingDone = true; + } +} + +template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) +{ + // set initial conditions + amrex::GpuArray const dx = grid_elem.dx_; + amrex::GpuArray prob_lo = grid_elem.prob_lo_; + amrex::GpuArray prob_hi = grid_elem.prob_hi_; + const amrex::Box &indexRange = grid_elem.indexRange_; + const amrex::Array4 &state_cc = grid_elem.array_; + + + std::array numdens = {-1.0}; + + for (int n = 1; n <= NumSpec; ++n) { + switch (n) { + + case 1: + numdens[n - 1] = userData_.primary_species_1; + break; + case 2: + numdens[n - 1] = userData_.primary_species_2; + break; + case 3: + numdens[n - 1] = userData_.primary_species_3; + break; + case 4: + numdens[n - 1] = userData_.primary_species_4; + break; + case 5: + numdens[n - 1] = userData_.primary_species_5; + break; + case 6: + numdens[n - 1] = userData_.primary_species_6; + break; + case 7: + numdens[n - 1] = userData_.primary_species_7; + break; + case 8: + numdens[n - 1] = userData_.primary_species_8; + break; + case 9: + numdens[n - 1] = userData_.primary_species_9; + break; + case 10: + numdens[n - 1] = userData_.primary_species_10; + break; + case 11: + numdens[n - 1] = userData_.primary_species_11; + break; + case 12: + numdens[n - 1] = userData_.primary_species_12; + break; + case 13: + numdens[n - 1] = userData_.primary_species_13; + break; + case 14: + numdens[n - 1] = userData_.primary_species_14; + break; + + default: + amrex::Abort("Cannot initialize number density for chem specie"); + break; + } + } + + + amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); + amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); + amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); + + // cloud parameters + const double R_sphere = userData_.R_sphere; + const double omega_sphere = userData_.omega_sphere; + const double renorm_amp = userData_.rescale_factor; + const double numdens_init = userData_.numdens_init; + const double core_temp = userData_.temperature; + + auto const &dvx = userData_.dvx.const_table(); + auto const &dvy = userData_.dvy.const_table(); + auto const &dvz = userData_.dvz.const_table(); + + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) { + amrex::Real const x = prob_lo[0] + (i + static_cast(0.5)) * dx[0]; + amrex::Real const y = prob_lo[1] + (j + static_cast(0.5)) * dx[1]; + amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx[2]; + amrex::Real const r = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2) + std::pow(z - z0, 2)); + amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); + + eos_t state; + amrex::Real rhotot = 0.0; + + for (int n = 0; n < NumSpec; ++n) { + state.xn[n] = numdens[n] * numdens_init; + rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS + } + + // normalize -- just in case + std::array mfracs = {-1.0}; + Real msum = 0.0; + for (int n = 0; n < NumSpec; ++n) { + mfracs[n] = state.xn[n] * spmasses[n] / rhotot; + msum += mfracs[n]; + } + + for (int n = 0; n < NumSpec; ++n) { + mfracs[n] /= msum; + // use the normalized mass fractions to obtain the corresponding number densities + state.xn[n] = mfracs[n] * rhotot / spmasses[n]; + } + + //amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; + + amrex::Real phi = atan2((y-y0), (x-x0)); + + double vx = renorm_amp * dvx(i, j, k); + double vy = renorm_amp * dvy(i, j, k); + double vz = renorm_amp * dvz(i, j, k); + + amrex::Real pres = 0.0; + + if (r <= R_sphere) { + state.rho = rhotot; + state.T = core_temp; + eos(eos_input_rt, state); + + // add rotation to vx and vy + vx += (-1.0) * distxy * omega_sphere * std::sin(phi); + vy += distxy * omega_sphere * std::cos(phi); + + } else { + state.rho = 0.01 * rhotot; + state.p = 3.595730e-10; //pressure equilibrium - this is the pressure within the core + eos(eos_input_rp, state); + + } + + // call the EOS to set initial internal energy e + amrex::Real e = state.rho * state.e; + + //amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; + + state_cc(i, j, k, HydroSystem::density_index) = state.rho; + state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; + state_cc(i, j, k, HydroSystem::x2Momentum_index) = state.rho * vy; + state_cc(i, j, k, HydroSystem::x3Momentum_index) = state.rho * vz; + state_cc(i, j, k, HydroSystem::internalEnergy_index) = e; + + Real const Egas = RadSystem::ComputeEgasFromEint(state.rho, state.rho*vx, state.rho*vy, state.rho*vz, e); + state_cc(i, j, k, HydroSystem::energy_index) = Egas; + + for (int nn = 0; nn < NumSpec; ++nn) { + state_cc(i, j, k, HydroSystem::scalar0_index + nn) = + mfracs[nn] * state.rho; // we use partial densities and not mass fractions + } + + }); +} + + +template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) +{ + // refine on Jeans length + const int N_cells = 32; // inverse of the 'Jeans number' [Truelove et al. (1997)] + const amrex::Real G = Gconst_; + const amrex::Real dx = geom[lev].CellSizeArray()[0]; + + for (amrex::MFIter mfi(state_new_cc_[lev]); mfi.isValid(); ++mfi) { + const amrex::Box &box = mfi.validbox(); + const auto state = state_new_cc_[lev].const_array(mfi); + const auto tag = tags.array(mfi); + const int nidx = HydroSystem::density_index; + + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + + Real const rho = state(i, j, k, nidx); + Real pressure = HydroSystem::ComputePressure(state, i, j, k); + amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state, i, j, k); + + amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + + const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); + + if (l_Jeans < (N_cells * dx) && rho > 5e-20) { + tag(i, j, k) = amrex::TagBox::SET; + + } + }); + } + +} + + +template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const +{ + // compute derived variables and save in 'mf' + if (dname == "temperature") { + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + amrex::Real Eint = state[bx](i, j, k, HydroSystem::internalEnergy_index); + + amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + + output[bx](i, j, k, ncomp) = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); + }); + } + + if (dname == "pressure") { + + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + amrex::Real Pgas = HydroSystem::ComputePressure(state[bx], i, j, k); + output[bx](i, j, k, ncomp) = Pgas; + }); + } + + + if (dname == "velx") { + + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + Real const xmom = state[bx](i, j, k, HydroSystem::x1Momentum_index); + output[bx](i, j, k, ncomp) = xmom / rho; + }); + } + +} + +auto problem_main() -> int +{ + // read problem parameters + amrex::ParmParse const pp("perturb"); + + // cloud radius + Real R_sphere{}; + pp.query("cloud_radius", R_sphere); + + // cloud density + Real numdens_init{}; + pp.query("cloud_numdens", numdens_init); + + // cloud angular velocity + Real omega_sphere{}; + pp.query("cloud_omega", omega_sphere); + + + // boundary conditions + const int ncomp_cc = Physics_Indices::nvarTotal_cc; + amrex::Vector BCs_cc(ncomp_cc); + for (int n = 0; n < ncomp_cc; ++n) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + BCs_cc[n].setLo(i, amrex::BCType::foextrap); + BCs_cc[n].setHi(i, amrex::BCType::foextrap); + } + } + + // Problem initialization + RadhydroSimulation sim(BCs_cc); + sim.doPoissonSolve_ = 1; // enable self-gravity + sim.densityFloor_ = 1e-25; + sim.tempFloor_ = 2.73 * (30.0 + 1.0); + sim.speedCeiling_ = 3e6; + + sim.userData_.R_sphere = R_sphere; + sim.userData_.numdens_init = numdens_init; + sim.userData_.omega_sphere = omega_sphere; + + sim.initDt_ = 1e6; + + // initialize + sim.setInitialConditions(); + + // evolve + sim.evolve(); + + int const status = 0; + return status; +} diff --git a/src/CopIII/copiii.hpp b/src/CopIII/copiii.hpp new file mode 100644 index 000000000..175488e24 --- /dev/null +++ b/src/CopIII/copiii.hpp @@ -0,0 +1,23 @@ +#ifndef COPIII_HPP_ // NOLINT +#define COPIII_HPP_ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file star_cluster.hpp +/// \brief Defines a test problem for Pop III star formation. +/// Author: Piyush Sharda (Leiden University, 2023) +/// + +// external headers +#include + +// internal headers +#include "hydro_system.hpp" +#include "interpolate.hpp" + +// function definitions +auto problem_main() -> int; + +#endif // COPIII_HPP_ From c805e1faf1dc3734350c77deb5a311afc0436cbb Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 22:24:14 +0800 Subject: [PATCH 081/165] fix bug --- src/CopIII/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CopIII/CMakeLists.txt b/src/CopIII/CMakeLists.txt index 9205bc8d6..82ba3b10f 100644 --- a/src/CopIII/CMakeLists.txt +++ b/src/CopIII/CMakeLists.txt @@ -10,10 +10,10 @@ if (AMReX_SPACEDIM EQUAL 3) include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H") add_executable(copiii copiii.cpp ../TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) - target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM + target_compile_definitions(copiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM if(AMReX_GPU_BACKEND MATCHES "CUDA") - setup_target_for_cuda_compilation(popiii) + setup_target_for_cuda_compilation(copiii) endif() #need h5py to run perturbations.py file below From 21585237b193a7fa622bd285a9971aba14bc5c9c Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 22 Sep 2023 04:59:46 +0800 Subject: [PATCH 082/165] updated --- tests/PopIII.in | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index 52e1d5671..57904ea72 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -4,7 +4,7 @@ geometry.prob_lo = -3.703e18 -3.703e18 -3.703e18 geometry.prob_hi = 3.703e18 3.703e18 3.703e18 geometry.is_periodic = 0 0 0 - +#restartfile = "chk02050" # ***************************************************************** # VERBOSITY # ***************************************************************** @@ -16,11 +16,11 @@ amr.v = 1 # verbosity in Amr amr.n_cell = 128 128 128 amr.max_level = 10 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this -amr.max_grid_size = 128 # at least 128 for GPUs +amr.max_grid_size = 1024 # at least 128 for GPUs amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells amr.grid_eff = 0.7 # default -hydro.reconstruction_order = 3 # PPM +hydro.reconstruction_order = 2 # PPM cfl = 0.2 max_timesteps = 100000 stop_time = 1e16 @@ -28,9 +28,9 @@ stop_time = 1e16 do_reflux = 1 do_subcycle = 0 -#ascent_interval = 10 +#ascent_interval = 50 plotfile_interval = 50 #100 -checkpoint_interval = 100 +checkpoint_interval = 50 perturb.cloud_radius = 3.086e18 perturb.cloud_omega = 2.016008E-14 @@ -40,7 +40,7 @@ perturb.cloud_numdens = 0.90861183E+004 # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" -derived_vars = temperature +derived_vars = temperature velx pressure amrex.throw_exception = 0 amrex.signal_handling = 1 From f5024ae37f72903ddf462aad3e379273a76599f0 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 22 Sep 2023 05:03:12 +0800 Subject: [PATCH 083/165] reset refinement --- src/PopIII/popiii.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 20276f91f..9f1dc26f7 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -370,10 +370,8 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); - - if (l_Jeans < (N_cells * dx) && rho > 5e-20 && x > -6.172e17 && x < 1.543e18 && y > -6.172e17) { + if (l_Jeans < (N_cells * dx) && rho > 2e-20) { tag(i, j, k) = amrex::TagBox::SET; - } }); } From 4bce75636d572c114f9f269582084788e8189656 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Thu, 21 Sep 2023 23:07:18 +0200 Subject: [PATCH 084/165] added sound speed --- src/PopIII/popiii.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 9f1dc26f7..32823b9bd 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -425,6 +425,26 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str }); } + + if (dname == "sound_speed") { + + const int ncomp = ncomp_cc_in; + auto const &state = state_new_cc_[lev].const_arrays(); + auto output = mf.arrays(); + + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + + Real const rho = state[bx](i, j, k, HydroSystem::density_index); + Real pressure = HydroSystem::ComputePressure(state[bx], i, j, k); + amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + + amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + output[bx](i, j, k, ncomp) = cs; + }); + + } + + } auto problem_main() -> int From 6af6e2e561a03edc32faad23aab1689d2c9f1680 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Fri, 22 Sep 2023 05:26:36 +0800 Subject: [PATCH 085/165] added sound speed --- tests/PopIII.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index 57904ea72..37855850e 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -40,7 +40,7 @@ perturb.cloud_numdens = 0.90861183E+004 # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" -derived_vars = temperature velx pressure +derived_vars = temperature velx pressure sound_speed amrex.throw_exception = 0 amrex.signal_handling = 1 From 7817e1c979e51d382b5799f194b3290de894656e Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 22 Sep 2023 22:23:35 +0200 Subject: [PATCH 086/165] do not abort --- src/RadhydroSimulation.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 1cb935a8f..e58c4acb3 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -737,9 +737,9 @@ template void RadhydroSimulation::FixupState(int amrex::ParallelDescriptor::Barrier(); amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_old_cc_[lev], componentNames_cc_, geom[lev], 0, istep[lev] + 1); amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::ParallelDescriptor::Abort(); - } + //if (amrex::ParallelDescriptor::IOProcessor()) { + // amrex::ParallelDescriptor::Abort(); + //} } From f44ad77db87fc5002af86fcb04d96d6f2e753580 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 23 Sep 2023 15:32:39 +0200 Subject: [PATCH 087/165] refine on 64 cells --- src/PopIII/popiii.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 32823b9bd..1b96062d7 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -339,7 +339,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) { // refine on Jeans length - const int N_cells = 32; // inverse of the 'Jeans number' [Truelove et al. (1997)] + const int N_cells = 64; // inverse of the 'Jeans number' [Truelove et al. (1997)] const amrex::Real G = Gconst_; const amrex::Real dx = geom[lev].CellSizeArray()[0]; From 9b6596397aed794057302f06a7c608a4a2b2d5d5 Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 24 Sep 2023 15:17:48 +0200 Subject: [PATCH 088/165] update amrex --- extern/Microphysics | 2 +- extern/amrex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/Microphysics b/extern/Microphysics index ced7cc41d..9c08d809b 160000 --- a/extern/Microphysics +++ b/extern/Microphysics @@ -1 +1 @@ -Subproject commit ced7cc41dbafae11ed587c1d446d80e442b9b967 +Subproject commit 9c08d809b6798df9798691174e2fc3a64c41360c diff --git a/extern/amrex b/extern/amrex index 1aea95c02..2e9962813 160000 --- a/extern/amrex +++ b/extern/amrex @@ -1 +1 @@ -Subproject commit 1aea95c021a8cbf0cab5e9e81f02a1b6e66249b5 +Subproject commit 2e99628138df3b5b0ecf50b0c1201d5547f821a0 From 79c50f9f81c0dee9060f28a0f3483f1957a5d35e Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 24 Sep 2023 15:21:49 +0200 Subject: [PATCH 089/165] delete copiii --- src/CopIII/CMakeLists.txt | 75 ------ src/CopIII/ascent_actions.yaml | 30 --- src/CopIII/copiii.cpp | 469 --------------------------------- src/CopIII/copiii.hpp | 23 -- 4 files changed, 597 deletions(-) delete mode 100644 src/CopIII/CMakeLists.txt delete mode 100644 src/CopIII/ascent_actions.yaml delete mode 100644 src/CopIII/copiii.cpp delete mode 100644 src/CopIII/copiii.hpp diff --git a/src/CopIII/CMakeLists.txt b/src/CopIII/CMakeLists.txt deleted file mode 100644 index 82ba3b10f..000000000 --- a/src/CopIII/CMakeLists.txt +++ /dev/null @@ -1,75 +0,0 @@ -if (AMReX_SPACEDIM EQUAL 3) - # Define a custom target that runs the Python script to produce the input perturbations file - - - set(microphysics_network_name "primordial_chem") #this will override network_name to primordial_chem for this directory only - setup_target_for_microphysics_compilation(${microphysics_network_name} "${CMAKE_CURRENT_BINARY_DIR}/") - - #use the BEFORE keyword so that these files get priority in compilation for targets in this directory - #this is critical to ensure the correct Microphysics files are linked to primordial chem target - include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H") - - add_executable(copiii copiii.cpp ../TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) - target_compile_definitions(copiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM - - if(AMReX_GPU_BACKEND MATCHES "CUDA") - setup_target_for_cuda_compilation(copiii) - endif() - - #need h5py to run perturbations.py file below - execute_process( - COMMAND Python3::Interpreter -c "h5py" - RESULT_VARIABLE EXIT_CODE - OUTPUT_QUIET - ) - - #re-run cmake if this file is changed - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/CopIII.in) - - # Read the 'PopIII.in' file line by line - file(READ ${CMAKE_SOURCE_DIR}/tests/CopIII.in pop_in_contents) - - # Split the file contents into lines - string(REPLACE "\n" ";" pop_in_lines "${pop_in_contents}") - - # Iterate through the lines and look for lines starting with 'amr.n_cell' - foreach(line IN LISTS pop_in_lines) - if (line MATCHES "^amr\\.n_cell.*") - set(match_line "${line}") - break() - endif() - endforeach() - - if (match_line) - message(STATUS "picked line is ${match_line}") - # Use a regular expression to find consecutive digit - string(REGEX MATCHALL "[0-9]+" digits_list "${match_line}") - # Iterate through the matched digits and extract the first one - list(GET digits_list 0 first_digits) - # Check if matched digits were found - if (first_digits) - #first_digits give the box size, but you also want box size / 2 as kmax in the script below - math(EXPR int_first_digits "${first_digits}") - # Check if the conversion was successful - if (int_first_digits) - # Divide the integer by two - math(EXPR result "${int_first_digits} / 2") - else() - message(FATAL_ERROR "Conversion to integer failed, so cannot find kmax!") - endif() - message(STATUS "Will create initial perturbations of box size ${first_digits} and kmax ${result}") - - else() - message(FATAL_ERROR "No box size found to create initial perturbations for!") - endif() - else() - message(FATAL_ERROR "No matching line found in ${CMAKE_SOURCE_DIR}/tests/CopIII.in!") - endif() - - - add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=1.8 --f_solenoidal=0.66667 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) - add_test(NAME CopIII COMMAND copiii CopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) - set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP CopIII_fixture) - set_tests_properties(CopIII PROPERTIES FIXTURES_REQUIRED CopIII_fixture) - -endif() diff --git a/src/CopIII/ascent_actions.yaml b/src/CopIII/ascent_actions.yaml deleted file mode 100644 index 6515c4b84..000000000 --- a/src/CopIII/ascent_actions.yaml +++ /dev/null @@ -1,30 +0,0 @@ -- - action: "add_pipelines" - pipelines: - pl1: - f2: - type: "slice" - params: - point: - x: 0. - y: 0. - z: 0. - normal: - x: 0.0 - y: 0.0 - z: 1.0 -- - action: "add_scenes" - scenes: - s1: - plots: - p1: - type: "pseudocolor" - field: "log_density" - pipeline: "pl1" - renders: - r1: - image_prefix: "dens%05d" - annotations: "true" - camera: - zoom: 1.5 diff --git a/src/CopIII/copiii.cpp b/src/CopIII/copiii.cpp deleted file mode 100644 index 8cc05f54d..000000000 --- a/src/CopIII/copiii.cpp +++ /dev/null @@ -1,469 +0,0 @@ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file copiii.cpp -/// \brief Defines a test problem for Pop III star formation. -/// Author: Piyush Sharda (Leiden University, 2023) -/// -#include -#include -#include - -#include "AMReX.H" -#include "AMReX_Arena.H" -#include "AMReX_BC_TYPES.H" -#include "AMReX_BLassert.H" -#include "AMReX_Config.H" -#include "AMReX_FabArrayUtility.H" -#include "AMReX_MultiFab.H" -#include "AMReX_ParallelDescriptor.H" -#include "AMReX_ParmParse.H" -#include "AMReX_Print.H" -#include "AMReX_SPACE.H" -#include "AMReX_TableData.H" - -#include "RadhydroSimulation.hpp" -#include "SimulationData.hpp" -#include "TurbDataReader.hpp" -#include "hydro_system.hpp" -#include "copiii.hpp" - -#include "actual_eos_data.H" -#include "burn_type.H" -#include "eos.H" -#include "extern_parameters.H" -#include "network.H" - -using amrex::Real; - -struct CopIII { -}; - -template <> struct HydroSystem_Traits { - static constexpr bool reconstruct_eint = true; -}; - -template <> struct Physics_Traits { - // cell-centred - static constexpr bool is_hydro_enabled = true; - static constexpr int numMassScalars = NumSpec; // number of chemical species - static constexpr int numPassiveScalars = numMassScalars + 0; // we only have mass scalars - static constexpr bool is_radiation_enabled = false; - // face-centred - static constexpr bool is_mhd_enabled = false; -}; - - -template <> struct SimulationData { - // real-space perturbation fields - amrex::TableData dvx; - amrex::TableData dvy; - amrex::TableData dvz; - amrex::Real dv_rms_generated{}; - amrex::Real dv_rms_target{}; - amrex::Real rescale_factor{}; - - // cloud parameters - amrex::Real R_sphere{}; - amrex::Real numdens_init{}; - amrex::Real omega_sphere{}; - - amrex::Real small_temp{}; - amrex::Real small_dens{}; - amrex::Real temperature{}; - amrex::Real primary_species_1{}; - amrex::Real primary_species_2{}; - amrex::Real primary_species_3{}; - amrex::Real primary_species_4{}; - amrex::Real primary_species_5{}; - amrex::Real primary_species_6{}; - amrex::Real primary_species_7{}; - amrex::Real primary_species_8{}; - amrex::Real primary_species_9{}; - amrex::Real primary_species_10{}; - amrex::Real primary_species_11{}; - amrex::Real primary_species_12{}; - amrex::Real primary_species_13{}; - amrex::Real primary_species_14{}; - - -}; - -template <> void RadhydroSimulation::preCalculateInitialConditions() -{ - - // initialize microphysics routines - init_extern_parameters(); - - // parmparse species and temperature - amrex::ParmParse pp("primordial_chem"); - userData_.small_temp = 1e1; - pp.query("small_temp", userData_.small_temp); - - userData_.small_dens = 1e-60; - pp.query("small_dens", userData_.small_dens); - - userData_.temperature = 1e1; - pp.query("temperature", userData_.temperature); - - userData_.primary_species_1 = 1.0e0_rt; - userData_.primary_species_2 = 0.0e0_rt; - userData_.primary_species_3 = 0.0e0_rt; - userData_.primary_species_4 = 0.0e0_rt; - userData_.primary_species_5 = 0.0e0_rt; - userData_.primary_species_6 = 0.0e0_rt; - userData_.primary_species_7 = 0.0e0_rt; - userData_.primary_species_8 = 0.0e0_rt; - userData_.primary_species_9 = 0.0e0_rt; - userData_.primary_species_10 = 0.0e0_rt; - userData_.primary_species_11 = 0.0e0_rt; - userData_.primary_species_12 = 0.0e0_rt; - userData_.primary_species_13 = 0.0e0_rt; - userData_.primary_species_14 = 0.0e0_rt; - - pp.query("primary_species_1", userData_.primary_species_1); - pp.query("primary_species_2", userData_.primary_species_2); - pp.query("primary_species_3", userData_.primary_species_3); - pp.query("primary_species_4", userData_.primary_species_4); - pp.query("primary_species_5", userData_.primary_species_5); - pp.query("primary_species_6", userData_.primary_species_6); - pp.query("primary_species_7", userData_.primary_species_7); - pp.query("primary_species_8", userData_.primary_species_8); - pp.query("primary_species_9", userData_.primary_species_9); - pp.query("primary_species_10", userData_.primary_species_10); - pp.query("primary_species_11", userData_.primary_species_11); - pp.query("primary_species_12", userData_.primary_species_12); - pp.query("primary_species_13", userData_.primary_species_13); - pp.query("primary_species_14", userData_.primary_species_14); - - eos_init(userData_.small_temp, userData_.small_dens); - network_init(); - - - static bool isSamplingDone = false; - if (!isSamplingDone) { - // read perturbations from file - turb_data turbData; - amrex::ParmParse pp("perturb"); - std::string turbdata_filename; - pp.query("filename", turbdata_filename); - initialize_turbdata(turbData, turbdata_filename); - - // copy to pinned memory - auto pinned_dvx = get_tabledata(turbData.dvx); - auto pinned_dvy = get_tabledata(turbData.dvy); - auto pinned_dvz = get_tabledata(turbData.dvz); - - // compute normalisation - userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); - amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; - - const Real rms_dv_target = 1.8050e5; - const Real rms_dv_actual = userData_.dv_rms_generated; - userData_.rescale_factor = rms_dv_target / rms_dv_actual; - - // copy to GPU - userData_.dvx.resize(pinned_dvx.lo(), pinned_dvx.hi()); - userData_.dvx.copy(pinned_dvx); - - userData_.dvy.resize(pinned_dvy.lo(), pinned_dvy.hi()); - userData_.dvy.copy(pinned_dvy); - - userData_.dvz.resize(pinned_dvz.lo(), pinned_dvz.hi()); - userData_.dvz.copy(pinned_dvz); - - isSamplingDone = true; - } -} - -template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) -{ - // set initial conditions - amrex::GpuArray const dx = grid_elem.dx_; - amrex::GpuArray prob_lo = grid_elem.prob_lo_; - amrex::GpuArray prob_hi = grid_elem.prob_hi_; - const amrex::Box &indexRange = grid_elem.indexRange_; - const amrex::Array4 &state_cc = grid_elem.array_; - - - std::array numdens = {-1.0}; - - for (int n = 1; n <= NumSpec; ++n) { - switch (n) { - - case 1: - numdens[n - 1] = userData_.primary_species_1; - break; - case 2: - numdens[n - 1] = userData_.primary_species_2; - break; - case 3: - numdens[n - 1] = userData_.primary_species_3; - break; - case 4: - numdens[n - 1] = userData_.primary_species_4; - break; - case 5: - numdens[n - 1] = userData_.primary_species_5; - break; - case 6: - numdens[n - 1] = userData_.primary_species_6; - break; - case 7: - numdens[n - 1] = userData_.primary_species_7; - break; - case 8: - numdens[n - 1] = userData_.primary_species_8; - break; - case 9: - numdens[n - 1] = userData_.primary_species_9; - break; - case 10: - numdens[n - 1] = userData_.primary_species_10; - break; - case 11: - numdens[n - 1] = userData_.primary_species_11; - break; - case 12: - numdens[n - 1] = userData_.primary_species_12; - break; - case 13: - numdens[n - 1] = userData_.primary_species_13; - break; - case 14: - numdens[n - 1] = userData_.primary_species_14; - break; - - default: - amrex::Abort("Cannot initialize number density for chem specie"); - break; - } - } - - - amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); - amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); - amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); - - // cloud parameters - const double R_sphere = userData_.R_sphere; - const double omega_sphere = userData_.omega_sphere; - const double renorm_amp = userData_.rescale_factor; - const double numdens_init = userData_.numdens_init; - const double core_temp = userData_.temperature; - - auto const &dvx = userData_.dvx.const_table(); - auto const &dvy = userData_.dvy.const_table(); - auto const &dvz = userData_.dvz.const_table(); - - amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - amrex::Real const x = prob_lo[0] + (i + static_cast(0.5)) * dx[0]; - amrex::Real const y = prob_lo[1] + (j + static_cast(0.5)) * dx[1]; - amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx[2]; - amrex::Real const r = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2) + std::pow(z - z0, 2)); - amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); - - eos_t state; - amrex::Real rhotot = 0.0; - - for (int n = 0; n < NumSpec; ++n) { - state.xn[n] = numdens[n] * numdens_init; - rhotot += state.xn[n] * spmasses[n]; // spmasses contains the masses of all species, defined in EOS - } - - // normalize -- just in case - std::array mfracs = {-1.0}; - Real msum = 0.0; - for (int n = 0; n < NumSpec; ++n) { - mfracs[n] = state.xn[n] * spmasses[n] / rhotot; - msum += mfracs[n]; - } - - for (int n = 0; n < NumSpec; ++n) { - mfracs[n] /= msum; - // use the normalized mass fractions to obtain the corresponding number densities - state.xn[n] = mfracs[n] * rhotot / spmasses[n]; - } - - //amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; - - amrex::Real phi = atan2((y-y0), (x-x0)); - - double vx = renorm_amp * dvx(i, j, k); - double vy = renorm_amp * dvy(i, j, k); - double vz = renorm_amp * dvz(i, j, k); - - amrex::Real pres = 0.0; - - if (r <= R_sphere) { - state.rho = rhotot; - state.T = core_temp; - eos(eos_input_rt, state); - - // add rotation to vx and vy - vx += (-1.0) * distxy * omega_sphere * std::sin(phi); - vy += distxy * omega_sphere * std::cos(phi); - - } else { - state.rho = 0.01 * rhotot; - state.p = 3.595730e-10; //pressure equilibrium - this is the pressure within the core - eos(eos_input_rp, state); - - } - - // call the EOS to set initial internal energy e - amrex::Real e = state.rho * state.e; - - //amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; - - state_cc(i, j, k, HydroSystem::density_index) = state.rho; - state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; - state_cc(i, j, k, HydroSystem::x2Momentum_index) = state.rho * vy; - state_cc(i, j, k, HydroSystem::x3Momentum_index) = state.rho * vz; - state_cc(i, j, k, HydroSystem::internalEnergy_index) = e; - - Real const Egas = RadSystem::ComputeEgasFromEint(state.rho, state.rho*vx, state.rho*vy, state.rho*vz, e); - state_cc(i, j, k, HydroSystem::energy_index) = Egas; - - for (int nn = 0; nn < NumSpec; ++nn) { - state_cc(i, j, k, HydroSystem::scalar0_index + nn) = - mfracs[nn] * state.rho; // we use partial densities and not mass fractions - } - - }); -} - - -template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) -{ - // refine on Jeans length - const int N_cells = 32; // inverse of the 'Jeans number' [Truelove et al. (1997)] - const amrex::Real G = Gconst_; - const amrex::Real dx = geom[lev].CellSizeArray()[0]; - - for (amrex::MFIter mfi(state_new_cc_[lev]); mfi.isValid(); ++mfi) { - const amrex::Box &box = mfi.validbox(); - const auto state = state_new_cc_[lev].const_array(mfi); - const auto tag = tags.array(mfi); - const int nidx = HydroSystem::density_index; - - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - - Real const rho = state(i, j, k, nidx); - Real pressure = HydroSystem::ComputePressure(state, i, j, k); - amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state, i, j, k); - - amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); - - const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); - - if (l_Jeans < (N_cells * dx) && rho > 5e-20) { - tag(i, j, k) = amrex::TagBox::SET; - - } - }); - } - -} - - -template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const -{ - // compute derived variables and save in 'mf' - if (dname == "temperature") { - const int ncomp = ncomp_cc_in; - auto const &state = state_new_cc_[lev].const_arrays(); - auto output = mf.arrays(); - - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - - Real const rho = state[bx](i, j, k, HydroSystem::density_index); - amrex::Real Eint = state[bx](i, j, k, HydroSystem::internalEnergy_index); - - amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); - - output[bx](i, j, k, ncomp) = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); - }); - } - - if (dname == "pressure") { - - const int ncomp = ncomp_cc_in; - auto const &state = state_new_cc_[lev].const_arrays(); - auto output = mf.arrays(); - - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - amrex::Real Pgas = HydroSystem::ComputePressure(state[bx], i, j, k); - output[bx](i, j, k, ncomp) = Pgas; - }); - } - - - if (dname == "velx") { - - const int ncomp = ncomp_cc_in; - auto const &state = state_new_cc_[lev].const_arrays(); - auto output = mf.arrays(); - - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - - Real const rho = state[bx](i, j, k, HydroSystem::density_index); - Real const xmom = state[bx](i, j, k, HydroSystem::x1Momentum_index); - output[bx](i, j, k, ncomp) = xmom / rho; - }); - } - -} - -auto problem_main() -> int -{ - // read problem parameters - amrex::ParmParse const pp("perturb"); - - // cloud radius - Real R_sphere{}; - pp.query("cloud_radius", R_sphere); - - // cloud density - Real numdens_init{}; - pp.query("cloud_numdens", numdens_init); - - // cloud angular velocity - Real omega_sphere{}; - pp.query("cloud_omega", omega_sphere); - - - // boundary conditions - const int ncomp_cc = Physics_Indices::nvarTotal_cc; - amrex::Vector BCs_cc(ncomp_cc); - for (int n = 0; n < ncomp_cc; ++n) { - for (int i = 0; i < AMREX_SPACEDIM; ++i) { - BCs_cc[n].setLo(i, amrex::BCType::foextrap); - BCs_cc[n].setHi(i, amrex::BCType::foextrap); - } - } - - // Problem initialization - RadhydroSimulation sim(BCs_cc); - sim.doPoissonSolve_ = 1; // enable self-gravity - sim.densityFloor_ = 1e-25; - sim.tempFloor_ = 2.73 * (30.0 + 1.0); - sim.speedCeiling_ = 3e6; - - sim.userData_.R_sphere = R_sphere; - sim.userData_.numdens_init = numdens_init; - sim.userData_.omega_sphere = omega_sphere; - - sim.initDt_ = 1e6; - - // initialize - sim.setInitialConditions(); - - // evolve - sim.evolve(); - - int const status = 0; - return status; -} diff --git a/src/CopIII/copiii.hpp b/src/CopIII/copiii.hpp deleted file mode 100644 index 175488e24..000000000 --- a/src/CopIII/copiii.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef COPIII_HPP_ // NOLINT -#define COPIII_HPP_ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file star_cluster.hpp -/// \brief Defines a test problem for Pop III star formation. -/// Author: Piyush Sharda (Leiden University, 2023) -/// - -// external headers -#include - -// internal headers -#include "hydro_system.hpp" -#include "interpolate.hpp" - -// function definitions -auto problem_main() -> int; - -#endif // COPIII_HPP_ From 1317371cbfe7dc643db18a584166e7d030c9315a Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 24 Sep 2023 15:29:12 +0200 Subject: [PATCH 090/165] remove debugs --- src/RadhydroSimulation.hpp | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index e58c4acb3..b4f5fd883 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -433,11 +433,11 @@ template auto RadhydroSimulation::computeExtraPh return std::numeric_limits::max(); } -//#if !defined(NDEBUG) +#if !defined(NDEBUG) #define CHECK_HYDRO_STATES(mf) checkHydroStates(mf, __FILE__, __LINE__) -//#else -//#define CHECK_HYDRO_STATES(mf) -//#endif +#else +#define CHECK_HYDRO_STATES(mf) +#endif template void RadhydroSimulation::checkHydroStates(amrex::MultiFab &mf, char const *file, int line) { @@ -451,7 +451,6 @@ template void RadhydroSimulation::checkHydroStat amrex::Print() << "Writing checkpoint for debugging...\n"; amrex::MFIter::allowMultipleMFIters(true); WriteCheckpointFile(); - amrex::Abort("Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")"); } } @@ -545,8 +544,6 @@ template void RadhydroSimulation::computeAfterEv amrex::Real const Egas0 = initSumCons[RadSystem::gasEnergy_index]; amrex::Real const Egas = state_new_cc_[0].sum(RadSystem::gasEnergy_index) * vol; - amrex::Print() << "after evolve Egas " << Egas << " vol " << vol << std::endl; - amrex::Real Etot0 = NAN; amrex::Real Etot = NAN; if constexpr (Physics_Traits::is_radiation_enabled) { @@ -712,7 +709,6 @@ template void RadhydroSimulation::FixupState(int amrex::ParallelDescriptor::Barrier(); // fix hydro state - amrex::Print() << "enforce limits state_new_cc " << std::endl; bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev], "fixup_new"); amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); if (!enforchk) { @@ -722,12 +718,11 @@ template void RadhydroSimulation::FixupState(int amrex::ParallelDescriptor::Barrier(); amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], 0, istep[lev] + 1); amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::ParallelDescriptor::Abort(); - } + //if (amrex::ParallelDescriptor::IOProcessor()) { + // amrex::ParallelDescriptor::Abort(); + //} } - amrex::Print() << "enforce limits state_old_cc " << std::endl; enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev], "fixup_old"); amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); if (!enforchk) { @@ -1027,17 +1022,6 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // update ghost zones [old timestep] fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); - bool validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); - amrex::ParallelDescriptor::ReduceBoolAnd(validStates); - if (!validStates) { - amrex::Print() << "Hydro states invalid after fillBC in RK \n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid after fillBC in RK "); - } - - // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) if (lowLevelDebuggingOutput_ == 1) { #ifdef AMREX_USE_ASCENT From cd9657f5d36b4bae9c9d118515035b08950f1548 Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 24 Sep 2023 15:32:34 +0200 Subject: [PATCH 091/165] remove debugs --- src/RadhydroSimulation.hpp | 45 +++----------------------------------- 1 file changed, 3 insertions(+), 42 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index b4f5fd883..ae02b3cf3 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1042,18 +1042,6 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); - validStates = HydroSystem::CheckStatesValid(state_old_cc_tmp); - amrex::ParallelDescriptor::ReduceBoolAnd(validStates); - - if (!validStates) { - amrex::Print() << "Hydro states invalid before RK2 stage 1 \n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid before RK2 stage 1 "); - } - - // Stage 1 of RK2-SSP { // advance all grids on local processor (Stage 1 of integrator) @@ -1142,7 +1130,6 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum - amrex::Print() << "enforce limits fofc 1 " << std::endl; bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew, "RK2-stage-1"); amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); if (!enforchk) { @@ -1152,9 +1139,9 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o amrex::ParallelDescriptor::Barrier(); amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateNew, componentNames_cc_, geom[lev], 0, istep[lev] + 1); amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::ParallelDescriptor::Abort(); - } + //if (amrex::ParallelDescriptor::IOProcessor()) { + // amrex::ParallelDescriptor::Abort(); + //} } @@ -1171,18 +1158,6 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } amrex::Gpu::streamSynchronizeAll(); - validStates = HydroSystem::CheckStatesValid(state_inter_cc_); - amrex::ParallelDescriptor::ReduceBoolAnd(validStates); - - if (!validStates) { - amrex::Print() << "Hydro states invalid after RK2 stage 1 \n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid after RK2 stage "); - } - - // Stage 2 of RK2-SSP if (integratorOrder_ == 2) { // update ghost zones [intermediate stage stored in state_inter_cc_] @@ -1252,21 +1227,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum - amrex::Print() << "enforce limits prevent vacuum stage 2 " << std::endl; bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal, "RK2-stage-2"); - amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); - if (!enforchk) { - amrex::Print() << "enforce limits has rho = 0 fixup prevent vacuum stage 2 rk2 " << std::endl; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::ParallelDescriptor::Barrier(); - amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateFinal, componentNames_cc_, geom[lev], 0, istep[lev] + 1); - amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::ParallelDescriptor::Abort(); - } - } - if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) From 9a12f0add59b53f0c23d6084fb326f13763fabec Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 24 Sep 2023 15:33:49 +0200 Subject: [PATCH 092/165] remove debugs --- src/RadhydroSimulation.hpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index ae02b3cf3..c096a3cdc 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -1131,20 +1131,6 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o // prevent vacuum bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew, "RK2-stage-1"); - amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); - if (!enforchk) { - amrex::Print() << "enforce limits has rho = 0 fixup fofc 1 stage 1 rk2 " << std::endl; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::ParallelDescriptor::Barrier(); - amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), stateNew, componentNames_cc_, geom[lev], 0, istep[lev] + 1); - amrex::ParallelDescriptor::Barrier(); - //if (amrex::ParallelDescriptor::IOProcessor()) { - // amrex::ParallelDescriptor::Abort(); - //} - } - - if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) From 7854b375a4019a5d983073c405d2835dfb627628 Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 24 Sep 2023 15:41:49 +0200 Subject: [PATCH 093/165] updated --- tests/PopIII.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index 37855850e..dc8303825 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -4,7 +4,7 @@ geometry.prob_lo = -3.703e18 -3.703e18 -3.703e18 geometry.prob_hi = 3.703e18 3.703e18 3.703e18 geometry.is_periodic = 0 0 0 -#restartfile = "chk02050" +#restartfile = "chk15360" # ***************************************************************** # VERBOSITY # ***************************************************************** @@ -13,15 +13,15 @@ amr.v = 1 # verbosity in Amr # ***************************************************************** # Resolution and refinement # ***************************************************************** -amr.n_cell = 128 128 128 +amr.n_cell = 64 64 64 amr.max_level = 10 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this amr.max_grid_size = 1024 # at least 128 for GPUs amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells amr.grid_eff = 0.7 # default -hydro.reconstruction_order = 2 # PPM -cfl = 0.2 +hydro.reconstruction_order = 3 # PPM +cfl = 0.15 max_timesteps = 100000 stop_time = 1e16 @@ -30,7 +30,7 @@ do_subcycle = 0 #ascent_interval = 50 plotfile_interval = 50 #100 -checkpoint_interval = 50 +checkpoint_interval = 200 perturb.cloud_radius = 3.086e18 perturb.cloud_omega = 2.016008E-14 From 604d95b9bd49d576c32b6fc2eeff6ff2804f7039 Mon Sep 17 00:00:00 2001 From: psharda Date: Mon, 25 Sep 2023 10:31:55 +0200 Subject: [PATCH 094/165] remove speed ceiling --- src/PopIII/popiii.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 1b96062d7..f8afd9b59 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -480,7 +480,7 @@ auto problem_main() -> int sim.doPoissonSolve_ = 1; // enable self-gravity sim.densityFloor_ = 1e-25; sim.tempFloor_ = 2.73 * (30.0 + 1.0); - sim.speedCeiling_ = 3e6; + // sim.speedCeiling_ = 3e6; sim.userData_.R_sphere = R_sphere; sim.userData_.numdens_init = numdens_init; From 2907834db3c533c1f9345038382c327b3762e012 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 25 Sep 2023 10:32:22 +0200 Subject: [PATCH 095/165] renormalize species post flooring --- src/hydro_system.hpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 21009031f..66e604250 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -780,11 +780,24 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex // Enforcing Limits on mass scalars if (nmscalars_ > 0) { - for (int idx = 0; idx < nmscalars_; ++idx) { - if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - state[bx](i, j, k, scalar0_index + idx) = small_x * rho; - } - } + amrex::Real sp_sum = NAN; + + for (int idx = 0; idx < nmscalars_; ++idx) { + + if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { + state[bx](i, j, k, scalar0_index + idx) = small_x * rho; + } + + // get sum to renormalize + sp_sum += state[bx](i, j, k, scalar0_index + idx); + + } + + for (int idx = 0; idx < nmscalars_; ++idx) { + // renormalize + state[bx](i, j, k, scalar0_index + idx) /= sp_sum; + } + } From 177e05bc6afd82f467f3977008a01119cf6fe6ca Mon Sep 17 00:00:00 2001 From: psharda Date: Mon, 25 Sep 2023 15:33:51 +0200 Subject: [PATCH 096/165] renormalize mass scalars in enforce limits --- src/hydro_system.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 66e604250..64101c101 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -780,12 +780,12 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex // Enforcing Limits on mass scalars if (nmscalars_ > 0) { - amrex::Real sp_sum = NAN; + amrex::Real sp_sum = 0.0; for (int idx = 0; idx < nmscalars_; ++idx) { if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - state[bx](i, j, k, scalar0_index + idx) = small_x * rho; + state[bx](i, j, k, scalar0_index + idx) = small_x * rho; } // get sum to renormalize @@ -793,12 +793,13 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex } + sp_sum /= rho; //get mass fractions + for (int idx = 0; idx < nmscalars_; ++idx) { // renormalize state[bx](i, j, k, scalar0_index + idx) /= sp_sum; } - } // Enforcing Limits on temperature estimated from Etot and Ekin From 322731cf600cace3734a38496556fb57a0503d1a Mon Sep 17 00:00:00 2001 From: psharda Date: Mon, 25 Sep 2023 15:42:51 +0200 Subject: [PATCH 097/165] resolve spell check --- src/PopIII/popiii.cpp | 2 -- src/hydro_system.hpp | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index f8afd9b59..a69b76a36 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -295,8 +295,6 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: double vy = renorm_amp * dvy(i, j, k); double vz = renorm_amp * dvz(i, j, k); - amrex::Real pres = 0.0; - if (r <= R_sphere) { state.rho = rhotot; state.T = core_temp; diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 64101c101..81f4c0a18 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -810,7 +810,7 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); if (std::isnan(P_star) || P_star < 0) { - printf("pres is %f", P_star); + printf("pressure is %f", P_star); } amrex::Real P_new = P_star; if (P_star < pressureFloor) { From f9aeffcb77225f667085dd72efaeb83537eb8eb0 Mon Sep 17 00:00:00 2001 From: psharda Date: Tue, 26 Sep 2023 11:55:41 +0200 Subject: [PATCH 098/165] dont need velocity large anymore in states valid --- src/hydro_system.hpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 81f4c0a18..06897aa13 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -441,15 +441,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am { // check if cons(i, j, k) is a valid state const amrex::Real rho = cons(i, j, k, density_index); - const amrex::Real momx = cons(i, j, k, x1Momentum_index); - const amrex::Real momy = cons(i, j, k, x2Momentum_index); - const amrex::Real momz = cons(i, j, k, x3Momentum_index); - const amrex::Real vel = std::pow(std::pow(momx/rho, 2) + std::pow(momy/rho, 2) + std::pow(momz/rho, 2), 0.5); bool isDensityPositive = (rho > 0.); - bool isVelocityLarge = (vel < 1e7); - bool isMassScalarPositive = true; if constexpr (nmscalars_ > 0) { amrex::GpuArray massScalars_ = RadSystem::ComputeMassScalars(cons, i, j, k); @@ -476,7 +470,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am #endif // return (isDensityPositive && isPressurePositive); - return isDensityPositive && isMassScalarPositive && isVelocityLarge; + return isDensityPositive && isMassScalarPositive; } template From f264f93bb9fd5651e910605c5dbaf15b5db39304 Mon Sep 17 00:00:00 2001 From: psharda Date: Tue, 26 Sep 2023 11:58:14 +0200 Subject: [PATCH 099/165] undo some stuff --- src/hydro_system.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 06897aa13..b11f4b015 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -430,7 +430,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::ComputeVelocity template AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::ComputeVelocityX3(amrex::Array4 const &cons, int i, int j, int k) -> amrex::Real -{ +{ amrex::Real const rho = cons(i, j, k, density_index); amrex::Real const vel_z = cons(i, j, k, x3Momentum_index) / rho; return vel_z; @@ -441,7 +441,6 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE auto HydroSystem::isStateValid(am { // check if cons(i, j, k) is a valid state const amrex::Real rho = cons(i, j, k, density_index); - bool isDensityPositive = (rho > 0.); bool isMassScalarPositive = true; @@ -868,9 +867,6 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex } }); - - amrex::Print() << "called by " << inputString << std::endl; - } template From ee2a5c907ee0072bb12e83645d2b7b802ef984d5 Mon Sep 17 00:00:00 2001 From: psharda Date: Tue, 26 Sep 2023 18:11:08 +0200 Subject: [PATCH 100/165] remove debug printf --- src/hydro_system.hpp | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index b11f4b015..c41d21a96 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -734,19 +734,6 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real Ekin = rho * vsq / 2.; amrex::Real rho_new = rho; - if (std::isnan(rho)) { - printf("rho = %e Eint = %e Ekin = %e v_abs = %e \n", rho, Eint, Ekin, v_abs); - if (nmscalars_ > 0) { - - for (int idx = 0; idx < nmscalars_; ++idx) { - if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - printf("nm is %e \n", state[bx](i, j, k, scalar0_index + idx)); - } - } - } - - } - if (rho < densityFloor) { rho_new = densityFloor; state[bx](i, j, k, density_index) = rho_new; @@ -803,7 +790,7 @@ auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); if (std::isnan(P_star) || P_star < 0) { - printf("pressure is %f", P_star); + printf("pressure is %f ", P_star); } amrex::Real P_new = P_star; if (P_star < pressureFloor) { From 6371065d3277e7d04d89d173548c5ef60518002a Mon Sep 17 00:00:00 2001 From: psharda Date: Tue, 26 Sep 2023 18:15:31 +0200 Subject: [PATCH 101/165] remove debug printf --- src/hydro_system.hpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index c41d21a96..1363e3d83 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -219,7 +219,6 @@ template auto HydroSystem::maxSignalSpeedLocal(a } else { cs = ComputeSoundSpeed(cons[bx], i, j, k); } - return {cs + abs_vel}; }); } @@ -275,8 +274,8 @@ template auto HydroSystem::CheckStatesValid(amre const auto thermal_energy = E - kinetic_energy; const auto P = ComputePressure(cons[bx], i, j, k); - bool negativeDensity = (rho < 0.); - bool negativePressure = (P < 0.); + bool negativeDensity = (rho <= 0.); + bool negativePressure = (P <= 0.); if constexpr (is_eos_isothermal()) { if (negativeDensity) { @@ -290,17 +289,6 @@ template auto HydroSystem::CheckStatesValid(amre // return {false}; //} - - - if (std::isnan(rho) || std::isnan(P)) { - printf("nan state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, - thermal_energy, P); - return {false}; - } else if (rho == 0.0 || P == 0.0) { - printf("zero state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, thermal_energy, P); - return {false}; - } - } return {true}; }); From ea32e0661fa9917fda495b18696112a53652f996 Mon Sep 17 00:00:00 2001 From: psharda Date: Tue, 26 Sep 2023 18:16:42 +0200 Subject: [PATCH 102/165] remove debug printf --- src/hydro_system.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 1363e3d83..94f60e658 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -283,11 +283,11 @@ template auto HydroSystem::CheckStatesValid(amre return {false}; } } else { - //if (negativeDensity || negativePressure) { - // printf("invalid state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, - // thermal_energy, P); - // return {false}; - //} + if (negativeDensity || negativePressure) { + printf("invalid state at (%d, %d, %d): rho %g, Etot %g, Eint %g, P %g\n", i, j, k, rho, E, + thermal_energy, P); + return {false}; + } } return {true}; From 8652fd36fb2823b1527617691150f71b2c363bbd Mon Sep 17 00:00:00 2001 From: psharda Date: Tue, 26 Sep 2023 18:17:31 +0200 Subject: [PATCH 103/165] remove debug printf --- src/hydro_system.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 94f60e658..b9fd99a34 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -288,7 +288,6 @@ template auto HydroSystem::CheckStatesValid(amre thermal_energy, P); return {false}; } - } return {true}; }); From 76e97702c7e1799e8f92a947be7f9b60363c39f0 Mon Sep 17 00:00:00 2001 From: psharda Date: Wed, 27 Sep 2023 16:50:22 +0200 Subject: [PATCH 104/165] revert enforceLimits as boolean --- src/hydro_system.hpp | 207 ++++++++++++++++++++----------------------- 1 file changed, 97 insertions(+), 110 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index b9fd99a34..9025cb354 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -108,8 +108,8 @@ template class HydroSystem : public HyperbolicSystem posvec) -> amrex::GpuArray; - static auto EnforceLimits(amrex::Real densityFloor, amrex::Real pressureFloor, amrex::Real speedCeiling, amrex::Real tempCeiling, - amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString) -> bool; + static void EnforceLimits(amrex::Real densityFloor, amrex::Real pressureFloor, amrex::Real speedCeiling, amrex::Real tempCeiling, + amrex::Real const tempFloor, amrex::MultiFab &state_mf); static void AddInternalEnergyPdV(amrex::MultiFab &rhs_mf, amrex::MultiFab const &consVar_mf, amrex::GpuArray dx, std::array const &faceVelArray, amrex::iMultiFab const &redoFlag_mf); @@ -700,146 +700,133 @@ void HydroSystem::FlattenShocks(amrex::MultiFab const &q_mf, amrex::M // to ensure that physical quantities are within reasonable // floors and ceilings which can be set in the param file template -auto HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex::Real const pressureFloor, amrex::Real const speedCeiling, - amrex::Real const tempCeiling, amrex::Real const tempFloor, amrex::MultiFab &state_mf, const char* inputString) -> bool +void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex::Real const pressureFloor, amrex::Real const speedCeiling, + amrex::Real const tempCeiling, amrex::Real const tempFloor, amrex::MultiFab &state_mf) { auto state = state_mf.arrays(); - amrex::Real strcall = 0.0; - return amrex::ParReduce(amrex::TypeList{}, amrex::TypeList{}, state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept -> amrex::GpuTuple { + amrex::ParallelFor(state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { amrex::Real const rho = state[bx](i, j, k, density_index); - if (rho != 0.0 && !std::isnan(rho)) { - amrex::Real const vx1 = state[bx](i, j, k, x1Momentum_index) / rho; - amrex::Real const vx2 = state[bx](i, j, k, x2Momentum_index) / rho; - amrex::Real const vx3 = state[bx](i, j, k, x3Momentum_index) / rho; - amrex::Real const vsq = (vx1 * vx1 + vx2 * vx2 + vx3 * vx3); - amrex::Real const v_abs = std::sqrt(vx1 * vx1 + vx2 * vx2 + vx3 * vx3); - amrex::Real Etot = state[bx](i, j, k, energy_index); - amrex::Real Eint = state[bx](i, j, k, internalEnergy_index); - amrex::Real Ekin = rho * vsq / 2.; - amrex::Real rho_new = rho; - - if (rho < densityFloor) { - rho_new = densityFloor; - state[bx](i, j, k, density_index) = rho_new; - state[bx](i, j, k, internalEnergy_index) = Eint * rho_new / rho; - state[bx](i, j, k, energy_index) = rho_new * vsq / 2. + (Etot - Ekin); - if (nscalars_ > 0) { - for (int n = 0; n < nscalars_; ++n) { - if (rho_new == 0.0) { - state[bx](i, j, k, scalar0_index + n) = 0.0; - } else { - state[bx](i, j, k, scalar0_index + n) *= rho / rho_new; - } + amrex::Real const vx1 = state[bx](i, j, k, x1Momentum_index) / rho; + amrex::Real const vx2 = state[bx](i, j, k, x2Momentum_index) / rho; + amrex::Real const vx3 = state[bx](i, j, k, x3Momentum_index) / rho; + amrex::Real const vsq = (vx1 * vx1 + vx2 * vx2 + vx3 * vx3); + amrex::Real const v_abs = std::sqrt(vx1 * vx1 + vx2 * vx2 + vx3 * vx3); + amrex::Real Etot = state[bx](i, j, k, energy_index); + amrex::Real Eint = state[bx](i, j, k, internalEnergy_index); + amrex::Real Ekin = rho * vsq / 2.; + amrex::Real rho_new = rho; + + if (rho < densityFloor) { + rho_new = densityFloor; + state[bx](i, j, k, density_index) = rho_new; + state[bx](i, j, k, internalEnergy_index) = Eint * rho_new / rho; + state[bx](i, j, k, energy_index) = rho_new * vsq / 2. + (Etot - Ekin); + if (nscalars_ > 0) { + for (int n = 0; n < nscalars_; ++n) { + if (rho_new == 0.0) { + state[bx](i, j, k, scalar0_index + n) = 0.0; + } else { + state[bx](i, j, k, scalar0_index + n) *= rho / rho_new; } } } + } - if (v_abs > speedCeiling) { - amrex::Real rescale_factor = speedCeiling / v_abs; - state[bx](i, j, k, x1Momentum_index) *= rescale_factor; - state[bx](i, j, k, x2Momentum_index) *= rescale_factor; - state[bx](i, j, k, x3Momentum_index) *= rescale_factor; - } - - // Enforcing Limits on mass scalars - if (nmscalars_ > 0) { - - amrex::Real sp_sum = 0.0; + if (v_abs > speedCeiling) { + amrex::Real rescale_factor = speedCeiling / v_abs; + state[bx](i, j, k, x1Momentum_index) *= rescale_factor; + state[bx](i, j, k, x2Momentum_index) *= rescale_factor; + state[bx](i, j, k, x3Momentum_index) *= rescale_factor; + } - for (int idx = 0; idx < nmscalars_; ++idx) { + // Enforcing Limits on mass scalars + if (nmscalars_ > 0) { - if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - state[bx](i, j, k, scalar0_index + idx) = small_x * rho; - } + amrex::Real sp_sum = 0.0; - // get sum to renormalize - sp_sum += state[bx](i, j, k, scalar0_index + idx); + for (int idx = 0; idx < nmscalars_; ++idx) { + if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { + state[bx](i, j, k, scalar0_index + idx) = small_x * rho; } - sp_sum /= rho; //get mass fractions - - for (int idx = 0; idx < nmscalars_; ++idx) { - // renormalize - state[bx](i, j, k, scalar0_index + idx) /= sp_sum; - } + // get sum to renormalize + sp_sum += state[bx](i, j, k, scalar0_index + idx); } - // Enforcing Limits on temperature estimated from Etot and Ekin - if (!HydroSystem::is_eos_isothermal()) { - // recompute gas energy (to prevent P < 0) - amrex::Real const Eint_star = Etot - 0.5 * rho_new * vsq; - amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); - amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); + sp_sum /= rho; //get mass fractions - if (std::isnan(P_star) || P_star < 0) { - printf("pressure is %f ", P_star); - } - amrex::Real P_new = P_star; - if (P_star < pressureFloor) { - P_new = pressureFloor; -#pragma nv_diag_suppress divide_by_zero - amrex::Real const Etot_new = quokka::EOS::ComputeEintFromPres(rho_new, P_new, massScalars) + 0.5 * rho_new * vsq; - state[bx](i, j, k, HydroSystem::energy_index) = Etot_new; - } - } - // re-obtain Ekin and Etot for putting limits on Temperature - if (rho_new == 0.0) { - Ekin = Etot = 0.0; - } else { - Ekin = std::pow(state[bx](i, j, k, x1Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; - Ekin += std::pow(state[bx](i, j, k, x2Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; - Ekin += std::pow(state[bx](i, j, k, x3Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; - Etot = state[bx](i, j, k, energy_index); + for (int idx = 0; idx < nmscalars_; ++idx) { + // renormalize + state[bx](i, j, k, scalar0_index + idx) /= sp_sum; } + } + + // Enforcing Limits on temperature estimated from Etot and Ekin + if (!HydroSystem::is_eos_isothermal()) { + // recompute gas energy (to prevent P < 0) + amrex::Real const Eint_star = Etot - 0.5 * rho_new * vsq; amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); - amrex::Real primTemp = quokka::EOS::ComputeTgasFromEint(rho, (Etot - Ekin), massScalars); + amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); - if (primTemp > tempCeiling) { - amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); - state[bx](i, j, k, energy_index) = Ekin + prim_eint; + if (std::isnan(P_star) || P_star < 0) { + printf("pressure is %f ", P_star); } - - if (primTemp < tempFloor) { - amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); - state[bx](i, j, k, energy_index) = Ekin + prim_eint; + amrex::Real P_new = P_star; + if (P_star < pressureFloor) { + P_new = pressureFloor; +#pragma nv_diag_suppress divide_by_zero + amrex::Real const Etot_new = quokka::EOS::ComputeEintFromPres(rho_new, P_new, massScalars) + 0.5 * rho_new * vsq; + state[bx](i, j, k, HydroSystem::energy_index) = Etot_new; } + } + // re-obtain Ekin and Etot for putting limits on Temperature + if (rho_new == 0.0) { + Ekin = Etot = 0.0; + } else { + Ekin = std::pow(state[bx](i, j, k, x1Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; + Ekin += std::pow(state[bx](i, j, k, x2Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; + Ekin += std::pow(state[bx](i, j, k, x3Momentum_index), 2.) / state[bx](i, j, k, density_index) / 2.; + Etot = state[bx](i, j, k, energy_index); + } + amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); + amrex::Real primTemp = quokka::EOS::ComputeTgasFromEint(rho, (Etot - Ekin), massScalars); - // Enforcing Limits on Auxiliary temperature estimated from Eint - Eint = state[bx](i, j, k, internalEnergy_index); - amrex::Real auxTemp = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); + if (primTemp > tempCeiling) { + amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); + state[bx](i, j, k, energy_index) = Ekin + prim_eint; + } - if (auxTemp > tempCeiling) { - state[bx](i, j, k, internalEnergy_index) = - quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); - state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); - } + if (primTemp < tempFloor) { + amrex::Real prim_eint = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); + state[bx](i, j, k, energy_index) = Ekin + prim_eint; + } - if (auxTemp < tempFloor) { - state[bx](i, j, k, internalEnergy_index) = - quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); - state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); - } + // Enforcing Limits on Auxiliary temperature estimated from Eint + Eint = state[bx](i, j, k, internalEnergy_index); + amrex::Real auxTemp = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); - if (std::isnan(state[bx](i, j, k, density_index)) || state[bx](i, j, k, density_index) == 0.0) { - printf("rho out is nan or zero even though rho in is not nan %e !! ", state[bx](i, j, k, density_index)); - } + if (auxTemp > tempCeiling) { + state[bx](i, j, k, internalEnergy_index) = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); + state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); + } - return {true}; - } else if (std::isnan(rho)) { - //printf("rho in is nan howcome %e! \n", rho); - //strcall = 1.0; - return {false}; - } else { - //printf("rho in is zero woah %e! \n", rho); - //strcall = 1.0; - return {false}; + if (auxTemp < tempFloor) { + state[bx](i, j, k, internalEnergy_index) = + quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); + state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); + } + + if (std::isnan(state[bx](i, j, k, density_index)) || state[bx](i, j, k, density_index) == 0.0) { + printf("rho out is nan or zero even though rho in is not nan %e !! ", state[bx](i, j, k, density_index)); } + return {true}; + }); } From 7c3df931e2f1505003dcca09cbe07dc090e58a5b Mon Sep 17 00:00:00 2001 From: psharda Date: Wed, 27 Sep 2023 16:55:25 +0200 Subject: [PATCH 105/165] revert enforceLimits boolean --- src/RadhydroSimulation.hpp | 35 ++++------------------------------- src/hydro_system.hpp | 12 ------------ 2 files changed, 4 insertions(+), 43 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index c096a3cdc..a8f5b8b24 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -706,37 +706,10 @@ template void RadhydroSimulation::applyPoissonGr template void RadhydroSimulation::FixupState(int lev) { BL_PROFILE("RadhydroSimulation::FixupState()"); - amrex::ParallelDescriptor::Barrier(); // fix hydro state - bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev], "fixup_new"); - amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); - if (!enforchk) { - amrex::Print() << "enforce limits has rho = 0 fixup state_new_cc " << std::endl; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::ParallelDescriptor::Barrier(); - amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], 0, istep[lev] + 1); - amrex::ParallelDescriptor::Barrier(); - //if (amrex::ParallelDescriptor::IOProcessor()) { - // amrex::ParallelDescriptor::Abort(); - //} - } - - enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev], "fixup_old"); - amrex::ParallelDescriptor::ReduceBoolAnd(enforchk); - if (!enforchk) { - amrex::Print() << "enforce limits has rho = 0 fixup state_old_cc " << std::endl; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::ParallelDescriptor::Barrier(); - amrex::WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_old_cc_[lev], componentNames_cc_, geom[lev], 0, istep[lev] + 1); - amrex::ParallelDescriptor::Barrier(); - //if (amrex::ParallelDescriptor::IOProcessor()) { - // amrex::ParallelDescriptor::Abort(); - //} - } - + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); // sync internal energy and total energy HydroSystem::SyncDualEnergy(state_new_cc_[lev]); @@ -1130,7 +1103,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum - bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew, "RK2-stage-1"); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) @@ -1213,7 +1186,7 @@ auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_o } // prevent vacuum - bool enforchk = HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal, "RK2-stage-2"); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); if (useDualEnergy_ == 1) { // sync internal energy (requires positive density) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 9025cb354..1ca2ce67f 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -708,7 +708,6 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::ParallelFor(state_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { amrex::Real const rho = state[bx](i, j, k, density_index); - amrex::Real const vx1 = state[bx](i, j, k, x1Momentum_index) / rho; amrex::Real const vx2 = state[bx](i, j, k, x2Momentum_index) / rho; amrex::Real const vx3 = state[bx](i, j, k, x3Momentum_index) / rho; @@ -772,10 +771,6 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real const Eint_star = Etot - 0.5 * rho_new * vsq; amrex::GpuArray massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); amrex::Real const P_star = quokka::EOS::ComputePressure(rho_new, Eint_star, massScalars); - - if (std::isnan(P_star) || P_star < 0) { - printf("pressure is %f ", P_star); - } amrex::Real P_new = P_star; if (P_star < pressureFloor) { P_new = pressureFloor; @@ -820,13 +815,6 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempFloor, massScalars); state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); } - - if (std::isnan(state[bx](i, j, k, density_index)) || state[bx](i, j, k, density_index) == 0.0) { - printf("rho out is nan or zero even though rho in is not nan %e !! ", state[bx](i, j, k, density_index)); - } - - return {true}; - }); } From c04e3c433c76af01bcb015e474d41743bac1b381 Mon Sep 17 00:00:00 2001 From: psharda Date: Wed, 27 Sep 2023 16:57:09 +0200 Subject: [PATCH 106/165] fix spaces --- src/RadhydroSimulation.hpp | 1 - src/hydro_system.hpp | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index a8f5b8b24..2b26a4800 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -710,7 +710,6 @@ template void RadhydroSimulation::FixupState(int // fix hydro state HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); - // sync internal energy and total energy HydroSystem::SyncDualEnergy(state_new_cc_[lev]); } diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 1ca2ce67f..8f3a982a3 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -806,7 +806,8 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real auxTemp = quokka::EOS::ComputeTgasFromEint(rho, Eint, massScalars); if (auxTemp > tempCeiling) { - state[bx](i, j, k, internalEnergy_index) = quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); + state[bx](i, j, k, internalEnergy_index) = + quokka::EOS::ComputeEintFromTgas(state[bx](i, j, k, density_index), tempCeiling, massScalars); state[bx](i, j, k, energy_index) = Ekin + state[bx](i, j, k, internalEnergy_index); } From 7bf9c792d89477a15971d4470b84335d04a1379a Mon Sep 17 00:00:00 2001 From: psharda Date: Wed, 27 Sep 2023 17:18:04 +0200 Subject: [PATCH 107/165] sample input file for main repo --- tests/PopIII.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index dc8303825..4f1b5b5a7 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -14,7 +14,7 @@ amr.v = 1 # verbosity in Amr # Resolution and refinement # ***************************************************************** amr.n_cell = 64 64 64 -amr.max_level = 10 # number of levels = max_level + 1 +amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this amr.max_grid_size = 1024 # at least 128 for GPUs amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells @@ -22,7 +22,7 @@ amr.grid_eff = 0.7 # default hydro.reconstruction_order = 3 # PPM cfl = 0.15 -max_timesteps = 100000 +max_timesteps = 10 stop_time = 1e16 do_reflux = 1 From 83d97359c07c6f28b72ec49817cf4f724e9591b2 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Wed, 27 Sep 2023 19:21:08 +0200 Subject: [PATCH 108/165] renormalize mass scalars post enforcing limit --- src/hydro_system.hpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 542197c5b..d707374e5 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -744,12 +744,20 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex // Enforcing Limits on mass scalars if (nmscalars_ > 0) { - for (int idx = 0; idx < nmscalars_; ++idx) { - if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { - state[bx](i, j, k, scalar0_index + idx) = small_x * rho; - } - } + amrex::Real sp_sum = 0.0; + for (int idx = 0; idx < nmscalars_; ++idx) { + if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { + state[bx](i, j, k, scalar0_index + idx) = small_x * rho; + } + // get sum to renormalize + sp_sum += state[bx](i, j, k, scalar0_index + idx); + } + sp_sum /= rho; //get mass fractions + for (int idx = 0; idx < nmscalars_; ++idx) { + // renormalize + state[bx](i, j, k, scalar0_index + idx) /= sp_sum; + } } // Enforcing Limits on temperature estimated from Etot and Ekin From 6e7edab010fc495c394e28d43af982c9e2716477 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 19:33:14 +0000 Subject: [PATCH 109/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/hydro_system.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 9eb773844..38ef5c047 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -752,9 +752,9 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex } if (sp_sum > 0) { - sp_sum /= rho; //get mass fractions + sp_sum /= rho; // get mass fractions for (int idx = 0; idx < nmscalars_; ++idx) { - // renormalize + // renormalize state[bx](i, j, k, scalar0_index + idx) /= sp_sum; } } From bf916aa74fc973fd6904b7d91345bfd3fc3fce1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 19:41:47 +0000 Subject: [PATCH 110/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PopIII/popiii.cpp | 35 ++++++----------------------------- src/hydro_system.hpp | 3 +-- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index a69b76a36..c5341129d 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -55,7 +55,6 @@ template <> struct Physics_Traits { static constexpr bool is_mhd_enabled = false; }; - template <> struct SimulationData { // real-space perturbation fields amrex::TableData dvx; @@ -87,8 +86,6 @@ template <> struct SimulationData { amrex::Real primary_species_12{}; amrex::Real primary_species_13{}; amrex::Real primary_species_14{}; - - }; template <> void RadhydroSimulation::preCalculateInitialConditions() @@ -141,7 +138,6 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() eos_init(userData_.small_temp, userData_.small_dens); network_init(); - static bool isSamplingDone = false; if (!isSamplingDone) { // read perturbations from file @@ -187,7 +183,6 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: const amrex::Box &indexRange = grid_elem.indexRange_; const amrex::Array4 &state_cc = grid_elem.array_; - std::array numdens = {-1.0}; for (int n = 1; n <= NumSpec; ++n) { @@ -242,7 +237,6 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: } } - amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); @@ -287,9 +281,9 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state.xn[n] = mfracs[n] * rhotot / spmasses[n]; } - //amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; + // amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; - amrex::Real phi = atan2((y-y0), (x-x0)); + amrex::Real phi = atan2((y - y0), (x - x0)); double vx = renorm_amp * dvx(i, j, k); double vy = renorm_amp * dvy(i, j, k); @@ -306,15 +300,14 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: } else { state.rho = 0.01 * rhotot; - state.p = 3.595730e-10; //pressure equilibrium - this is the pressure within the core + state.p = 3.595730e-10; // pressure equilibrium - this is the pressure within the core eos(eos_input_rp, state); - } // call the EOS to set initial internal energy e amrex::Real e = state.rho * state.e; - //amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; + // amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; state_cc(i, j, k, HydroSystem::density_index) = state.rho; state_cc(i, j, k, HydroSystem::x1Momentum_index) = state.rho * vx; @@ -322,18 +315,15 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: state_cc(i, j, k, HydroSystem::x3Momentum_index) = state.rho * vz; state_cc(i, j, k, HydroSystem::internalEnergy_index) = e; - Real const Egas = RadSystem::ComputeEgasFromEint(state.rho, state.rho*vx, state.rho*vy, state.rho*vz, e); + Real const Egas = RadSystem::ComputeEgasFromEint(state.rho, state.rho * vx, state.rho * vy, state.rho * vz, e); state_cc(i, j, k, HydroSystem::energy_index) = Egas; for (int nn = 0; nn < NumSpec; ++nn) { - state_cc(i, j, k, HydroSystem::scalar0_index + nn) = - mfracs[nn] * state.rho; // we use partial densities and not mass fractions + state_cc(i, j, k, HydroSystem::scalar0_index + nn) = mfracs[nn] * state.rho; // we use partial densities and not mass fractions } - }); } - template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) { // refine on Jeans length @@ -348,7 +338,6 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); - for (amrex::MFIter mfi(state_new_cc_[lev]); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.validbox(); const auto state = state_new_cc_[lev].const_array(mfi); @@ -356,7 +345,6 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra const int nidx = HydroSystem::density_index; amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real const x = prob_lo[0] + (i + static_cast(0.5)) * dx; amrex::Real const y = prob_lo[1] + (j + static_cast(0.5)) * dx; amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx; @@ -373,10 +361,8 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra } }); } - } - template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp_cc_in) const { // compute derived variables and save in 'mf' @@ -386,7 +372,6 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str auto output = mf.arrays(); amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - Real const rho = state[bx](i, j, k, HydroSystem::density_index); amrex::Real Eint = state[bx](i, j, k, HydroSystem::internalEnergy_index); @@ -408,7 +393,6 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str }); } - if (dname == "velx") { const int ncomp = ncomp_cc_in; @@ -416,14 +400,12 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str auto output = mf.arrays(); amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - Real const rho = state[bx](i, j, k, HydroSystem::density_index); Real const xmom = state[bx](i, j, k, HydroSystem::x1Momentum_index); output[bx](i, j, k, ncomp) = xmom / rho; }); } - if (dname == "sound_speed") { const int ncomp = ncomp_cc_in; @@ -431,7 +413,6 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str auto output = mf.arrays(); amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - Real const rho = state[bx](i, j, k, HydroSystem::density_index); Real pressure = HydroSystem::ComputePressure(state[bx], i, j, k); amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); @@ -439,10 +420,7 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); output[bx](i, j, k, ncomp) = cs; }); - } - - } auto problem_main() -> int @@ -462,7 +440,6 @@ auto problem_main() -> int Real omega_sphere{}; pp.query("cloud_omega", omega_sphere); - // boundary conditions const int ncomp_cc = Physics_Indices::nvarTotal_cc; amrex::Vector BCs_cc(ncomp_cc); diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 564057a4b..43b489e8a 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -751,10 +751,9 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex // get sum to renormalize sp_sum += state[bx](i, j, k, scalar0_index + idx); - } - sp_sum /= rho; //get mass fractions + sp_sum /= rho; // get mass fractions for (int idx = 0; idx < nmscalars_; ++idx) { // renormalize From 589638dfa6e3079a48c4f80fac5111e33ae16e9f Mon Sep 17 00:00:00 2001 From: psharda Date: Fri, 6 Oct 2023 21:43:11 +0200 Subject: [PATCH 111/165] update --- src/PopIII/popiii.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index c5341129d..9a1392834 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -53,6 +53,7 @@ template <> struct Physics_Traits { static constexpr bool is_radiation_enabled = false; // face-centred static constexpr bool is_mhd_enabled = false; + static constexpr int nGroups = 1; }; template <> struct SimulationData { From 03535f3bb52491b1c26b39eb0531eec3c268133e Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 7 Oct 2023 00:26:46 +0200 Subject: [PATCH 112/165] cosmetic --- src/hydro_system.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hydro_system.hpp b/src/hydro_system.hpp index 4d5c79a7a..38ef5c047 100644 --- a/src/hydro_system.hpp +++ b/src/hydro_system.hpp @@ -743,7 +743,6 @@ void HydroSystem::EnforceLimits(amrex::Real const densityFloor, amrex amrex::Real sp_sum = 0.0; for (int idx = 0; idx < nmscalars_; ++idx) { - if (state[bx](i, j, k, scalar0_index + idx) < 0.0) { state[bx](i, j, k, scalar0_index + idx) = small_x * rho; } From d074ff61135b89717e12c51890a5d471d0ec3270 Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 8 Oct 2023 00:52:32 +0200 Subject: [PATCH 113/165] remove actual_eos_data.H --- src/PopIII/popiii.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 9a1392834..f3c4a25ee 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -30,7 +30,6 @@ #include "hydro_system.hpp" #include "popiii.hpp" -#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From 0c3426330d49224e186c14d67a8154186c81fa1f Mon Sep 17 00:00:00 2001 From: psharda Date: Sun, 8 Oct 2023 01:12:48 +0200 Subject: [PATCH 114/165] resolve clang tidy comments --- src/PopIII/popiii.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index f3c4a25ee..433e7a255 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -30,6 +30,7 @@ #include "hydro_system.hpp" #include "popiii.hpp" +#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" @@ -95,7 +96,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() init_extern_parameters(); // parmparse species and temperature - amrex::ParmParse pp("primordial_chem"); + amrex::ParmParse const pp("primordial_chem"); userData_.small_temp = 1e1; pp.query("small_temp", userData_.small_temp); @@ -142,7 +143,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() if (!isSamplingDone) { // read perturbations from file turb_data turbData; - amrex::ParmParse pp("perturb"); + amrex::ParmParse const pp("perturb"); std::string turbdata_filename; pp.query("filename", turbdata_filename); initialize_turbdata(turbData, turbdata_filename); @@ -260,7 +261,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); eos_t state; - amrex::Real rhotot = 0.0; + amrex::Real const rhotot = 0.0; for (int n = 0; n < NumSpec; ++n) { state.xn[n] = numdens[n] * numdens_init; @@ -283,11 +284,11 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: // amrex::Print() << "cell " << i << j << k << " " << rhotot << " " << numdens_init << " " << numdens[0] << std::endl; - amrex::Real phi = atan2((y - y0), (x - x0)); + amrex::Real const phi = atan2((y - y0), (x - x0)); double vx = renorm_amp * dvx(i, j, k); double vy = renorm_amp * dvy(i, j, k); - double vz = renorm_amp * dvz(i, j, k); + double const vz = renorm_amp * dvz(i, j, k); if (r <= R_sphere) { state.rho = rhotot; @@ -305,7 +306,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: } // call the EOS to set initial internal energy e - amrex::Real e = state.rho * state.e; + amrex::Real const e = state.rho * state.e; // amrex::Print() << "cell " << i << j << k << " " << state.rho << " " << state.T << " " << e << std::endl; @@ -350,7 +351,7 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra amrex::Real const z = prob_lo[2] + (k + static_cast(0.5)) * dx; Real const rho = state(i, j, k, nidx); - Real pressure = HydroSystem::ComputePressure(state, i, j, k); + Real const pressure = HydroSystem::ComputePressure(state, i, j, k); amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state, i, j, k); amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); From 50825cb767e76158adabec8ac8a3120cd7197dce Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 9 Oct 2023 14:02:43 +0200 Subject: [PATCH 115/165] updated microp --- extern/Microphysics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/Microphysics b/extern/Microphysics index 9c08d809b..657406dca 160000 --- a/extern/Microphysics +++ b/extern/Microphysics @@ -1 +1 @@ -Subproject commit 9c08d809b6798df9798691174e2fc3a64c41360c +Subproject commit 657406dcaf546a5601c8388ef1a2b34a0016edb5 From 96a0a4b6ace2045524263bd20213734f9f1b7741 Mon Sep 17 00:00:00 2001 From: psharda Date: Sat, 4 Nov 2023 18:30:38 +0100 Subject: [PATCH 116/165] fix bug --- src/PopIII/popiii.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 433e7a255..371da70ae 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -261,7 +261,7 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: amrex::Real const distxy = std::sqrt(std::pow(x - x0, 2) + std::pow(y - y0, 2)); eos_t state; - amrex::Real const rhotot = 0.0; + amrex::Real rhotot = 0.0; for (int n = 0; n < NumSpec; ++n) { state.xn[n] = numdens[n] * numdens_init; From 0c2f6ca286cec4d423997c5d3738fed37107d8aa Mon Sep 17 00:00:00 2001 From: psharda Date: Wed, 20 Dec 2023 21:22:37 +0100 Subject: [PATCH 117/165] updated microp --- extern/Microphysics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/Microphysics b/extern/Microphysics index 657406dca..7de212422 160000 --- a/extern/Microphysics +++ b/extern/Microphysics @@ -1 +1 @@ -Subproject commit 657406dcaf546a5601c8388ef1a2b34a0016edb5 +Subproject commit 7de212422501a88c1c1f536b93217721e3dc6701 From 069444b58ae73cf02a8da42d4e5f649569fb75fe Mon Sep 17 00:00:00 2001 From: psharda Date: Thu, 25 Jan 2024 11:29:15 +0100 Subject: [PATCH 118/165] fix bug --- src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e2080cd82..bcf194785 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -180,5 +180,4 @@ add_subdirectory(FCQuantities) add_subdirectory(SphericalCollapse) add_subdirectory(NSCBC) add_subdirectory(StarCluster) -add_subdirectory(PrimordialChem) add_subdirectory(BinaryOrbitCIC) From 7af9e1abee9d4a149c9fc908a253dbd7880d6b10 Mon Sep 17 00:00:00 2001 From: psharda Date: Thu, 25 Jan 2024 11:32:04 +0100 Subject: [PATCH 119/165] fix bug --- src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bcf194785..43ef58004 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -179,5 +179,4 @@ add_subdirectory(PassiveScalar) add_subdirectory(FCQuantities) add_subdirectory(SphericalCollapse) add_subdirectory(NSCBC) -add_subdirectory(StarCluster) add_subdirectory(BinaryOrbitCIC) From 70424b38a759918c98e8f6dfc6f48941b2944ab7 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 13:23:21 +0100 Subject: [PATCH 120/165] dummy commit --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 8e6409dd6..c4cc467cd 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -38,7 +38,7 @@ using amrex::Real; struct PrimordialChemTest { -}; // dummy type to allow compile-type polymorphism via template specialization +}; // dummy type to allow compile-type polymorphism via template specializationnnnnn template <> struct Physics_Traits { // cell-centred From b87ae84ea64de0133adadaa28f30878ad349ed64 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 13:57:10 +0100 Subject: [PATCH 121/165] debug 1 --- .github/workflows/clang-tidy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 3e2fd32e8..6496fd811 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -25,6 +25,7 @@ jobs: apt_packages: libopenmpi-dev,libhdf5-mpi-dev,python3-dev,python3-numpy,python3-matplotlib cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF split_workflow: true + extra_args: src/PrimordialChem/test_primordial_chem.cpp # Uploads an artefact containing clang_fixes.json - uses: ZedThree/clang-tidy-review/upload@v0.17.1 From 24394a766d03a5549334ac71b2981e92ae50cfae Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 14:00:55 +0100 Subject: [PATCH 122/165] debug 1 --- .github/workflows/clang-tidy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 6496fd811..38a2296e7 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -25,7 +25,7 @@ jobs: apt_packages: libopenmpi-dev,libhdf5-mpi-dev,python3-dev,python3-numpy,python3-matplotlib cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF split_workflow: true - extra_args: src/PrimordialChem/test_primordial_chem.cpp + extra_args: '-- src/PrimordialChem/test_primordial_chem.cpp' # Uploads an artefact containing clang_fixes.json - uses: ZedThree/clang-tidy-review/upload@v0.17.1 From 1536cd7f79f62c129e129a782514be85a43d7ae9 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 14:18:18 +0100 Subject: [PATCH 123/165] include other headers present in primordial chem --- src/PopIII/popiii.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 371da70ae..012a4eb5e 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -9,25 +9,33 @@ /// #include #include +#include #include +#include #include "AMReX.H" #include "AMReX_Arena.H" #include "AMReX_BC_TYPES.H" +#include "AMReX_BLProfiler.H" #include "AMReX_BLassert.H" #include "AMReX_Config.H" +#include "AMReX_FabArray.H" #include "AMReX_FabArrayUtility.H" +#include "AMReX_GpuDevice.H" #include "AMReX_MultiFab.H" +#include "AMReX_ParallelContext.H" #include "AMReX_ParallelDescriptor.H" #include "AMReX_ParmParse.H" #include "AMReX_Print.H" +#include "AMReX_REAL.H" #include "AMReX_SPACE.H" #include "AMReX_TableData.H" #include "RadhydroSimulation.hpp" #include "SimulationData.hpp" -#include "TurbDataReader.hpp" #include "hydro_system.hpp" +#include "radiation_system.hpp" +#include "TurbDataReader.hpp" #include "popiii.hpp" #include "actual_eos_data.H" From e2e9aea6f14908449ee9200e1cf405cc442d1850 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 14:18:32 +0100 Subject: [PATCH 124/165] include openPMRsources as in primordial chem --- src/PopIII/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index dfbe99112..3d6f3f984 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -9,7 +9,7 @@ if (AMReX_SPACEDIM EQUAL 3) #this is critical to ensure the correct Microphysics files are linked to primordial chem target include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H") - add_executable(popiii popiii.cpp ../TurbDataReader.cpp ../main.cpp ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) + add_executable(popiii popiii.cpp ../TurbDataReader.cpp ../main.cpp "${openPMDSources}" ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources}) target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM if(AMReX_GPU_BACKEND MATCHES "CUDA") From c4c0def4a999c11e21aa2b50b207e1b0d02740ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:18:46 +0000 Subject: [PATCH 125/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PopIII/popiii.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 012a4eb5e..fe69cfdfc 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -7,9 +7,9 @@ /// \brief Defines a test problem for Pop III star formation. /// Author: Piyush Sharda (Leiden University, 2023) /// +#include #include #include -#include #include #include @@ -33,10 +33,10 @@ #include "RadhydroSimulation.hpp" #include "SimulationData.hpp" -#include "hydro_system.hpp" -#include "radiation_system.hpp" #include "TurbDataReader.hpp" +#include "hydro_system.hpp" #include "popiii.hpp" +#include "radiation_system.hpp" #include "actual_eos_data.H" #include "burn_type.H" From bb45cf2f5b4e6c3d07499fd69c2d3b83085c0db3 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 14:25:57 +0100 Subject: [PATCH 126/165] test radhydrosim --- src/RadhydroSimulation.hpp | 2997 ++++++++++++++++++------------------ 1 file changed, 1500 insertions(+), 1497 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index ae1e5dee3..696978e92 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -75,368 +75,371 @@ template class RadhydroSimulation : public AMRSimulation { public: - using AMRSimulation::state_old_cc_; - using AMRSimulation::state_new_cc_; - using AMRSimulation::max_signal_speed_; - using AMRSimulation::state_old_fc_; - using AMRSimulation::state_new_fc_; - using AMRSimulation::TracerPC; - - using AMRSimulation::nghost_cc_; - using AMRSimulation::areInitialConditionsDefined_; - using AMRSimulation::BCs_cc_; - using AMRSimulation::BCs_fc_; - using AMRSimulation::componentNames_cc_; - using AMRSimulation::componentNames_fc_; - using AMRSimulation::cflNumber_; - using AMRSimulation::fillBoundaryConditions; - using AMRSimulation::CustomPlotFileName; - using AMRSimulation::geom; - using AMRSimulation::grids; - using AMRSimulation::dmap; - using AMRSimulation::istep; - using AMRSimulation::flux_reg_; - using AMRSimulation::incrementFluxRegisters; - using AMRSimulation::finest_level; - using AMRSimulation::finestLevel; - using AMRSimulation::do_reflux; - using AMRSimulation::do_tracers; - using AMRSimulation::Verbose; - using AMRSimulation::constantDt_; - using AMRSimulation::boxArray; - using AMRSimulation::DistributionMap; - using AMRSimulation::refRatio; - using AMRSimulation::cellUpdates_; - using AMRSimulation::CountCells; - using AMRSimulation::WriteCheckpointFile; - using AMRSimulation::GetData; - using AMRSimulation::FillPatchWithData; - using AMRSimulation::Gconst_; - - using AMRSimulation::densityFloor_; - using AMRSimulation::tempFloor_; - using AMRSimulation::tempCeiling_; - using AMRSimulation::speedCeiling_; - - SimulationData userData_; - - int enableCooling_ = 0; - int enableChemistry_ = 0; - Real max_density_allowed = std::numeric_limits::max(); - quokka::cooling::cloudy_tables cloudyTables_; - std::string coolingTableFilename_{}; - - static constexpr int nvarTotal_cc_ = Physics_Indices::nvarTotal_cc; - static constexpr int ncompHydro_ = HydroSystem::nvar_; // hydro - static constexpr int ncompHyperbolic_ = RadSystem::nvarHyperbolic_; - static constexpr int nstartHyperbolic_ = RadSystem::nstartHyperbolic_; - - amrex::Real radiationCflNumber_ = 0.3; - int maxSubsteps_ = 10; // maximum number of radiation subcycles per hydro step - - bool computeReferenceSolution_ = false; - amrex::Real errorNorm_ = NAN; - amrex::Real pressureFloor_ = 0.; - - int lowLevelDebuggingOutput_ = 0; // 0 == do nothing; 1 == output intermediate multifabs used in hydro each timestep (ONLY USE FOR DEBUGGING) - int integratorOrder_ = 2; // 1 == forward Euler; 2 == RK2-SSP (default) - int reconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) - int radiationReconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) - int useDualEnergy_ = 1; // 0 == disabled; 1 == use auxiliary internal energy equation (default) - int abortOnFofcFailure_ = 1; // 0 == keep going, 1 == abort hydro advance if FOFC fails - amrex::Real artificialViscosityK_ = 0.; // artificial viscosity coefficient (default == None) - - amrex::Long radiationCellUpdates_ = 0; // total number of radiation cell-updates - - // member functions - explicit RadhydroSimulation(amrex::Vector &BCs_cc, amrex::Vector &BCs_fc) : AMRSimulation(BCs_cc, BCs_fc) - { - initialize(); - } - - explicit RadhydroSimulation(amrex::Vector &BCs_cc) : AMRSimulation(BCs_cc) { initialize(); } - - inline void initialize() - { - defineComponentNames(); - // read in runtime parameters - readParmParse(); - // set gamma - amrex::ParmParse eos("eos"); - eos.add("eos_gamma", quokka::EOS_Traits::gamma); - // initialize Microphysics params - init_extern_parameters(); - // initialize Microphysics EOS - amrex::Real small_temp = 1e-10; - amrex::Real small_dens = 1e-100; - eos_init(small_temp, small_dens); - } - - [[nodiscard]] static auto getScalarVariableNames() -> std::vector; - void defineComponentNames(); - void readParmParse(); - - void checkHydroStates(amrex::MultiFab &mf, char const *file, int line); - void computeMaxSignalLocal(int level) override; - auto computeExtraPhysicsTimestep(int lev) -> amrex::Real override; - void preCalculateInitialConditions() override; - void setInitialConditionsOnGrid(quokka::grid grid_elem) override; - void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) override; - void createInitialParticles() override; - void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) override; - void computeAfterTimestep() override; - void computeAfterLevelAdvance(int lev, amrex::Real time, amrex::Real dt_lev, int /*ncycle*/); - void computeAfterEvolve(amrex::Vector &initSumCons) override; - void computeReferenceSolution(amrex::MultiFab &ref, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo); - - // compute derived variables - void ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, int ncomp) const override; - - // compute projected vars - [[nodiscard]] auto ComputeProjections(int dir) const -> std::unordered_map> override; - - // compute statistics - auto ComputeStatistics() -> std::map override; - - // fix-up states - void FixupState(int level) override; - - // implement FillPatch function - void FillPatch(int lev, amrex::Real time, amrex::MultiFab &mf, int icomp, int ncomp, quokka::centering cen, quokka::direction dir, - FillPatchType fptype) override; - - // functions to operate on state vector before/after interpolating between levels - static void PreInterpState(amrex::MultiFab &mf, int scomp, int ncomp); - static void PostInterpState(amrex::MultiFab &mf, int scomp, int ncomp); - - // compute axis-aligned 1D profile of user_f(x, y, z) - template auto computeAxisAlignedProfile(int axis, F const &user_f) -> amrex::Gpu::HostVector; - - // tag cells for refinement - void ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override; - - // fill rhs for Poisson solve - void fillPoissonRhsAtLevel(amrex::MultiFab &rhs, int lev) override; - - // add gravitational acceleration to hydro state - void applyPoissonGravityAtLevel(amrex::MultiFab const &phi, int lev, amrex::Real dt) override; - - void addFluxArrays(std::array &dstfluxes, std::array &srcfluxes, const int srccomp, - const int dstcomp); - - auto expandFluxArrays(std::array &fluxes, int nstartNew, int ncompNew) - -> std::array; - - void printCoordinates(int lev, const amrex::IntVect &cell_idx); - - void advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - - auto advanceHydroAtLevel(amrex::MultiFab &state_old_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, - amrex::Real time, amrex::Real dt_lev) -> bool; - - void addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); - void addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); - - auto isCflViolated(int lev, amrex::Real time, amrex::Real dt_actual) -> bool; - - // radiation subcycle - void swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew); - auto computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int; - void advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, - amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); - void advanceRadiationForwardEuler(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - void advanceRadiationMidpointRK2(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - - void subcycleRadiationAtLevel(int lev, amrex::Real time, amrex::Real dt_lev_hydro, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - - void operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, amrex::Real time, double dt, int stage, - amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, - amrex::GpuArray const &prob_hi); - - auto computeRadiationFluxes(amrex::Array4 const &consVar, const amrex::Box &indexRange, int nvars, - amrex::GpuArray dx) - -> std::tuple, std::array>; - - auto computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) - -> std::pair, std::array>; + using AMRSimulation::state_old_cc_; + using AMRSimulation::state_new_cc_; + using AMRSimulation::max_signal_speed_; + using AMRSimulation::state_old_fc_; + using AMRSimulation::state_new_fc_; + using AMRSimulation::TracerPC; + + using AMRSimulation::nghost_cc_; + using AMRSimulation::areInitialConditionsDefined_; + using AMRSimulation::BCs_cc_; + using AMRSimulation::BCs_fc_; + using AMRSimulation::componentNames_cc_; + using AMRSimulation::componentNames_fc_; + using AMRSimulation::cflNumber_; + using AMRSimulation::fillBoundaryConditions; + using AMRSimulation::CustomPlotFileName; + using AMRSimulation::geom; + using AMRSimulation::grids; + using AMRSimulation::dmap; + using AMRSimulation::istep; + using AMRSimulation::flux_reg_; + using AMRSimulation::incrementFluxRegisters; + using AMRSimulation::finest_level; + using AMRSimulation::finestLevel; + using AMRSimulation::do_reflux; + using AMRSimulation::do_tracers; + using AMRSimulation::Verbose; + using AMRSimulation::constantDt_; + using AMRSimulation::boxArray; + using AMRSimulation::DistributionMap; + using AMRSimulation::refRatio; + using AMRSimulation::cellUpdates_; + using AMRSimulation::CountCells; + using AMRSimulation::WriteCheckpointFile; + using AMRSimulation::GetData; + using AMRSimulation::FillPatchWithData; + using AMRSimulation::Gconst_; + + using AMRSimulation::densityFloor_; + using AMRSimulation::tempFloor_; + using AMRSimulation::tempCeiling_; + using AMRSimulation::speedCeiling_; + + SimulationData userData_; + + int enableCooling_ = 0; + int enableChemistry_ = 0; + Real max_density_allowed = std::numeric_limits::max(); + Real min_density_allowed = std::numeric_limits::min(); + + quokka::cooling::cloudy_tables cloudyTables_; + std::string coolingTableFilename_{}; + + static constexpr int nvarTotal_cc_ = Physics_Indices::nvarTotal_cc; + static constexpr int ncompHydro_ = HydroSystem::nvar_; // hydro + static constexpr int ncompHyperbolic_ = RadSystem::nvarHyperbolic_; + static constexpr int nstartHyperbolic_ = RadSystem::nstartHyperbolic_; + + amrex::Real radiationCflNumber_ = 0.3; + int maxSubsteps_ = 10; // maximum number of radiation subcycles per hydro step + + bool computeReferenceSolution_ = false; + amrex::Real errorNorm_ = NAN; + amrex::Real pressureFloor_ = 0.; + + int lowLevelDebuggingOutput_ = 0; // 0 == do nothing; 1 == output intermediate multifabs used in hydro each timestep (ONLY USE FOR DEBUGGING) + int integratorOrder_ = 2; // 1 == forward Euler; 2 == RK2-SSP (default) + int reconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) + int radiationReconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) + int useDualEnergy_ = 1; // 0 == disabled; 1 == use auxiliary internal energy equation (default) + int abortOnFofcFailure_ = 1; // 0 == keep going, 1 == abort hydro advance if FOFC fails + amrex::Real artificialViscosityK_ = 0.; // artificial viscosity coefficient (default == None) + + amrex::Long radiationCellUpdates_ = 0; // total number of radiation cell-updates + + // member functions + explicit RadhydroSimulation(amrex::Vector &BCs_cc, amrex::Vector &BCs_fc) : AMRSimulation(BCs_cc, BCs_fc) + { + initialize(); + } + + explicit RadhydroSimulation(amrex::Vector &BCs_cc) : AMRSimulation(BCs_cc) { initialize(); } + + inline void initialize() + { + defineComponentNames(); + // read in runtime parameters + readParmParse(); + // set gamma + amrex::ParmParse eos("eos"); + eos.add("eos_gamma", quokka::EOS_Traits::gamma); + // initialize Microphysics params + init_extern_parameters(); + // initialize Microphysics EOS + amrex::Real small_temp = 1e-10; + amrex::Real small_dens = 1e-100; + eos_init(small_temp, small_dens); + } + + [[nodiscard]] static auto getScalarVariableNames() -> std::vector; + void defineComponentNames(); + void readParmParse(); + + void checkHydroStates(amrex::MultiFab &mf, char const *file, int line); + void computeMaxSignalLocal(int level) override; + auto computeExtraPhysicsTimestep(int lev) -> amrex::Real override; + void preCalculateInitialConditions() override; + void setInitialConditionsOnGrid(quokka::grid grid_elem) override; + void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) override; + void createInitialParticles() override; + void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) override; + void computeAfterTimestep() override; + void computeAfterLevelAdvance(int lev, amrex::Real time, amrex::Real dt_lev, int /*ncycle*/); + void computeAfterEvolve(amrex::Vector &initSumCons) override; + void computeReferenceSolution(amrex::MultiFab &ref, amrex::GpuArray const &dx, + amrex::GpuArray const &prob_lo); + + // compute derived variables + void ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, int ncomp) const override; + + // compute projected vars + [[nodiscard]] auto ComputeProjections(int dir) const -> std::unordered_map> override; + + // compute statistics + auto ComputeStatistics() -> std::map override; + + // fix-up states + void FixupState(int level) override; + + // implement FillPatch function + void FillPatch(int lev, amrex::Real time, amrex::MultiFab &mf, int icomp, int ncomp, quokka::centering cen, quokka::direction dir, + FillPatchType fptype) override; + + // functions to operate on state vector before/after interpolating between levels + static void PreInterpState(amrex::MultiFab &mf, int scomp, int ncomp); + static void PostInterpState(amrex::MultiFab &mf, int scomp, int ncomp); + + // compute axis-aligned 1D profile of user_f(x, y, z) + template auto computeAxisAlignedProfile(int axis, F const &user_f) -> amrex::Gpu::HostVector; + + // tag cells for refinement + void ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override; + + // fill rhs for Poisson solve + void fillPoissonRhsAtLevel(amrex::MultiFab &rhs, int lev) override; + + // add gravitational acceleration to hydro state + void applyPoissonGravityAtLevel(amrex::MultiFab const &phi, int lev, amrex::Real dt) override; + + void addFluxArrays(std::array &dstfluxes, std::array &srcfluxes, const int srccomp, + const int dstcomp); + + auto expandFluxArrays(std::array &fluxes, int nstartNew, int ncompNew) + -> std::array; + + void printCoordinates(int lev, const amrex::IntVect &cell_idx); + + void advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + + auto advanceHydroAtLevel(amrex::MultiFab &state_old_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, + amrex::Real time, amrex::Real dt_lev) -> bool; + + void addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); + void addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); + + auto isCflViolated(int lev, amrex::Real time, amrex::Real dt_actual) -> bool; + + // radiation subcycle + void swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew); + auto computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int; + void advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, + amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); + void advanceRadiationForwardEuler(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + void advanceRadiationMidpointRK2(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + + void subcycleRadiationAtLevel(int lev, amrex::Real time, amrex::Real dt_lev_hydro, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + + void operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, amrex::Real time, double dt, int stage, + amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, + amrex::GpuArray const &prob_hi); + + auto computeRadiationFluxes(amrex::Array4 const &consVar, const amrex::Box &indexRange, int nvars, + amrex::GpuArray dx) + -> std::tuple, std::array>; + + auto computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) + -> std::pair, std::array>; - auto computeFOHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) - -> std::pair, std::array>; + auto computeFOHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) + -> std::pair, std::array>; - template - void fluxFunction(amrex::Array4 const &consState, amrex::FArrayBox &x1Flux, amrex::FArrayBox &x1FluxDiffusive, - const amrex::Box &indexRange, int nvars, amrex::GpuArray dx); + template + void fluxFunction(amrex::Array4 const &consState, amrex::FArrayBox &x1Flux, amrex::FArrayBox &x1FluxDiffusive, + const amrex::Box &indexRange, int nvars, amrex::GpuArray dx); - template - void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, - int ng_reconstruct, int nvars); + template + void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, + amrex::MultiFab &x1FaceVel, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, + int ng_reconstruct, int nvars); - template - void hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, int ng_reconstruct, int nvars); + template + void hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, + amrex::MultiFab &x1FaceVel, int ng_reconstruct, int nvars); - void replaceFluxes(std::array &fluxes, std::array &FOfluxes, - amrex::iMultiFab &redoFlag); + void replaceFluxes(std::array &fluxes, std::array &FOfluxes, + amrex::iMultiFab &redoFlag); }; template void RadhydroSimulation::defineComponentNames() { - // cell-centred - // add hydro state variables - if constexpr (Physics_Traits::is_hydro_enabled || Physics_Traits::is_radiation_enabled) { - std::vector hydroNames = {"gasDensity", "x-GasMomentum", "y-GasMomentum", "z-GasMomentum", "gasEnergy", "gasInternalEnergy"}; - componentNames_cc_.insert(componentNames_cc_.end(), hydroNames.begin(), hydroNames.end()); - } - // add passive scalar variables - if constexpr (Physics_Traits::numPassiveScalars > 0) { - std::vector scalarNames = getScalarVariableNames(); - componentNames_cc_.insert(componentNames_cc_.end(), scalarNames.begin(), scalarNames.end()); - } - // add radiation state variables - if constexpr (Physics_Traits::is_radiation_enabled) { - std::vector radNames = {}; - for (int i = 0; i < Physics_Traits::nGroups; ++i) { - radNames.push_back("radEnergy-Group" + std::to_string(i)); - radNames.push_back("x-RadFlux-Group" + std::to_string(i)); - radNames.push_back("y-RadFlux-Group" + std::to_string(i)); - radNames.push_back("z-RadFlux-Group" + std::to_string(i)); - } - componentNames_cc_.insert(componentNames_cc_.end(), radNames.begin(), radNames.end()); - } - - // face-centred - - // add face-centered velocities - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-velocity"}); - } - // add mhd state variables - if constexpr (Physics_Traits::is_mhd_enabled) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-BField"}); - } - } + // cell-centred + // add hydro state variables + if constexpr (Physics_Traits::is_hydro_enabled || Physics_Traits::is_radiation_enabled) { + std::vector hydroNames = {"gasDensity", "x-GasMomentum", "y-GasMomentum", "z-GasMomentum", "gasEnergy", "gasInternalEnergy"}; + componentNames_cc_.insert(componentNames_cc_.end(), hydroNames.begin(), hydroNames.end()); + } + // add passive scalar variables + if constexpr (Physics_Traits::numPassiveScalars > 0) { + std::vector scalarNames = getScalarVariableNames(); + componentNames_cc_.insert(componentNames_cc_.end(), scalarNames.begin(), scalarNames.end()); + } + // add radiation state variables + if constexpr (Physics_Traits::is_radiation_enabled) { + std::vector radNames = {}; + for (int i = 0; i < Physics_Traits::nGroups; ++i) { + radNames.push_back("radEnergy-Group" + std::to_string(i)); + radNames.push_back("x-RadFlux-Group" + std::to_string(i)); + radNames.push_back("y-RadFlux-Group" + std::to_string(i)); + radNames.push_back("z-RadFlux-Group" + std::to_string(i)); + } + componentNames_cc_.insert(componentNames_cc_.end(), radNames.begin(), radNames.end()); + } + + // face-centred + + // add face-centered velocities + for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-velocity"}); + } + // add mhd state variables + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-BField"}); + } + } } template auto RadhydroSimulation::getScalarVariableNames() -> std::vector { - // return vector of names for the passive scalars - // this can be specialized by the user to provide more descriptive names - // (these names are used to label the variables in the plotfiles) - - std::vector names; - int nscalars = HydroSystem::nscalars_; - names.reserve(nscalars); - for (int n = 0; n < nscalars; ++n) { - // write string 'scalar_1', etc. - names.push_back(fmt::format("scalar_{}", n)); - } - return names; + // return vector of names for the passive scalars + // this can be specialized by the user to provide more descriptive names + // (these names are used to label the variables in the plotfiles) + + std::vector names; + int nscalars = HydroSystem::nscalars_; + names.reserve(nscalars); + for (int n = 0; n < nscalars; ++n) { + // write string 'scalar_1', etc. + names.push_back(fmt::format("scalar_{}", n)); + } + return names; } template void RadhydroSimulation::readParmParse() { - // set hydro runtime parameters - { - amrex::ParmParse hpp("hydro"); - hpp.query("low_level_debugging_output", lowLevelDebuggingOutput_); - hpp.query("rk_integrator_order", integratorOrder_); - hpp.query("reconstruction_order", reconstructionOrder_); - hpp.query("use_dual_energy", useDualEnergy_); - hpp.query("abort_on_fofc_failure", abortOnFofcFailure_); - hpp.query("artificial_viscosity_coefficient", artificialViscosityK_); - } - - // set cooling runtime parameters - { - amrex::ParmParse hpp("cooling"); - int alwaysReadTables = 0; - hpp.query("enabled", enableCooling_); - hpp.query("read_tables_even_if_disabled", alwaysReadTables); - hpp.query("grackle_data_file", coolingTableFilename_); - - if ((enableCooling_ == 1) || (alwaysReadTables == 1)) { - // read Cloudy tables - amrex::Print() << "Reading Cloudy tables...\n"; - quokka::cooling::readCloudyData(coolingTableFilename_, cloudyTables_); - } - } + // set hydro runtime parameters + { + amrex::ParmParse hpp("hydro"); + hpp.query("low_level_debugging_output", lowLevelDebuggingOutput_); + hpp.query("rk_integrator_order", integratorOrder_); + hpp.query("reconstruction_order", reconstructionOrder_); + hpp.query("use_dual_energy", useDualEnergy_); + hpp.query("abort_on_fofc_failure", abortOnFofcFailure_); + hpp.query("artificial_viscosity_coefficient", artificialViscosityK_); + } + + // set cooling runtime parameters + { + amrex::ParmParse hpp("cooling"); + int alwaysReadTables = 0; + hpp.query("enabled", enableCooling_); + hpp.query("read_tables_even_if_disabled", alwaysReadTables); + hpp.query("grackle_data_file", coolingTableFilename_); + + if ((enableCooling_ == 1) || (alwaysReadTables == 1)) { + // read Cloudy tables + amrex::Print() << "Reading Cloudy tables...\n"; + quokka::cooling::readCloudyData(coolingTableFilename_, cloudyTables_); + } + } #ifdef PRIMORDIAL_CHEM - // set chemistry runtime parameters - { - amrex::ParmParse hpp("primordial_chem"); - hpp.query("enabled", enableChemistry_); - hpp.query("max_density_allowed", max_density_allowed); - } + // set chemistry runtime parameters + { + amrex::ParmParse hpp("primordial_chem"); + hpp.query("enabled", enableChemistry_); + hpp.query("max_density_allowed", max_density_allowed); // chemistry is not accurate for densities > 3e-6 + hpp.query("min_density_allowed", min_density_allowed); // don't do chemistry in cells with densities below the minimum density specified + } #endif - // set radiation runtime parameters - { - amrex::ParmParse rpp("radiation"); - rpp.query("reconstruction_order", radiationReconstructionOrder_); - rpp.query("cfl", radiationCflNumber_); - } + // set radiation runtime parameters + { + amrex::ParmParse rpp("radiation"); + rpp.query("reconstruction_order", radiationReconstructionOrder_); + rpp.query("cfl", radiationCflNumber_); + } } template auto RadhydroSimulation::computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int { - // compute radiation timestep - auto const &dx = geom[lev].CellSizeArray(); - amrex::Real c_hat = RadSystem::c_hat_; - amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); - amrex::Real dtrad_tmp = radiationCflNumber_ * (dx_min / c_hat); - int nsubSteps = std::ceil(dt_lev_hydro / dtrad_tmp); - return nsubSteps; + // compute radiation timestep + auto const &dx = geom[lev].CellSizeArray(); + amrex::Real c_hat = RadSystem::c_hat_; + amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); + amrex::Real dtrad_tmp = radiationCflNumber_ * (dx_min / c_hat); + int nsubSteps = std::ceil(dt_lev_hydro / dtrad_tmp); + return nsubSteps; } template void RadhydroSimulation::computeMaxSignalLocal(int const level) { - BL_PROFILE("RadhydroSimulation::computeMaxSignalLocal()"); - - // hydro: loop over local grids, compute CFL timestep - for (amrex::MFIter iter(state_new_cc_[level]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[level].const_array(iter); - auto const &maxSignal = max_signal_speed_[level].array(iter); - - if constexpr (Physics_Traits::is_hydro_enabled && !(Physics_Traits::is_radiation_enabled)) { - // hydro only - HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); - } else if constexpr (Physics_Traits::is_radiation_enabled) { - // radiation hydro, or radiation only - RadSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); - if constexpr (Physics_Traits::is_hydro_enabled) { - auto maxSignalHydroFAB = amrex::FArrayBox(indexRange); - auto const &maxSignalHydro = maxSignalHydroFAB.array(); - HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignalHydro, indexRange); - const int maxSubsteps = maxSubsteps_; - // ensure that we use the smaller of the two timesteps - amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real const maxSignalRadiation = maxSignal(i, j, k) / static_cast(maxSubsteps); - maxSignal(i, j, k) = std::max(maxSignalRadiation, maxSignalHydro(i, j, k)); - }); - } - } else { - // no physics modules enabled, why are we running? - amrex::Abort("At least one of hydro or radiation must be enabled! Cannot " - "compute a time step."); - } - } + BL_PROFILE("RadhydroSimulation::computeMaxSignalLocal()"); + + // hydro: loop over local grids, compute CFL timestep + for (amrex::MFIter iter(state_new_cc_[level]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateNew = state_new_cc_[level].const_array(iter); + auto const &maxSignal = max_signal_speed_[level].array(iter); + + if constexpr (Physics_Traits::is_hydro_enabled && !(Physics_Traits::is_radiation_enabled)) { + // hydro only + HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); + } else if constexpr (Physics_Traits::is_radiation_enabled) { + // radiation hydro, or radiation only + RadSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); + if constexpr (Physics_Traits::is_hydro_enabled) { + auto maxSignalHydroFAB = amrex::FArrayBox(indexRange); + auto const &maxSignalHydro = maxSignalHydroFAB.array(); + HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignalHydro, indexRange); + const int maxSubsteps = maxSubsteps_; + // ensure that we use the smaller of the two timesteps + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real const maxSignalRadiation = maxSignal(i, j, k) / static_cast(maxSubsteps); + maxSignal(i, j, k) = std::max(maxSignalRadiation, maxSignalHydro(i, j, k)); + }); + } + } else { + // no physics modules enabled, why are we running? + amrex::Abort("At least one of hydro or radiation must be enabled! Cannot " + "compute a time step."); + } + } } template auto RadhydroSimulation::computeExtraPhysicsTimestep(int const level) -> amrex::Real { - BL_PROFILE("RadhydroSimulation::computeExtraPhysicsTimestep()"); - // users can override this to enforce additional timestep constraints - return std::numeric_limits::max(); + BL_PROFILE("RadhydroSimulation::computeExtraPhysicsTimestep()"); + // users can override this to enforce additional timestep constraints + return std::numeric_limits::max(); } #if !defined(NDEBUG) @@ -447,300 +450,300 @@ template auto RadhydroSimulation::computeExtraPh template void RadhydroSimulation::checkHydroStates(amrex::MultiFab &mf, char const *file, int line) { - BL_PROFILE("RadhydroSimulation::checkHydroStates()"); - - bool validStates = HydroSystem::CheckStatesValid(mf); - amrex::ParallelDescriptor::ReduceBoolAnd(validStates); - - if (!validStates) { - amrex::Print() << "Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")\n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")"); - } + BL_PROFILE("RadhydroSimulation::checkHydroStates()"); + + bool validStates = HydroSystem::CheckStatesValid(mf); + amrex::ParallelDescriptor::ReduceBoolAnd(validStates); + + if (!validStates) { + amrex::Print() << "Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")\n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")"); + } } template void RadhydroSimulation::preCalculateInitialConditions() { - // default empty implementation - // user should implement using problem-specific template specialization + // default empty implementation + // user should implement using problem-specific template specialization } template void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) { - // default empty implementation - // user should implement using problem-specific template specialization + // default empty implementation + // user should implement using problem-specific template specialization } template void RadhydroSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) { - // default empty implementation - // user should implement using problem-specific template specialization - // note: an implementation is only required if face-centered vars are used + // default empty implementation + // user should implement using problem-specific template specialization + // note: an implementation is only required if face-centered vars are used } template void RadhydroSimulation::createInitialParticles() { - // default empty implementation - // user should implement using problem-specific template specialization - // note: an implementation is only required if particles are used + // default empty implementation + // user should implement using problem-specific template specialization + // note: an implementation is only required if particles are used } template void RadhydroSimulation::computeAfterTimestep() { - // do nothing -- user should implement if desired + // do nothing -- user should implement if desired } template void RadhydroSimulation::computeAfterLevelAdvance(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) { - // user should implement if desired + // user should implement if desired } template void RadhydroSimulation::addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt) { - // user should implement - // (when Strang splitting is enabled, dt is actually 0.5*dt_lev) + // user should implement + // (when Strang splitting is enabled, dt is actually 0.5*dt_lev) } template void RadhydroSimulation::addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt) { - if (enableCooling_ == 1) { - // compute cooling - quokka::cooling::computeCooling(state, dt, cloudyTables_, tempFloor_); - } + if (enableCooling_ == 1) { + // compute cooling + quokka::cooling::computeCooling(state, dt, cloudyTables_, tempFloor_); + } #ifdef PRIMORDIAL_CHEM - if (enableChemistry_ == 1) { - // compute chemistry - quokka::chemistry::computeChemistry(state, dt, max_density_allowed); - } + if (enableChemistry_ == 1) { + // compute chemistry + quokka::chemistry::computeChemistry(state, dt, max_density_allowed, min_density_allowed); + } #endif - // compute user-specified sources - addStrangSplitSources(state, lev, time, dt); + // compute user-specified sources + addStrangSplitSources(state, lev, time, dt); } template void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp) const { - // compute derived variables and save in 'mf' -- user should implement + // compute derived variables and save in 'mf' -- user should implement } template auto RadhydroSimulation::ComputeProjections(int /*dir*/) const -> std::unordered_map> { - // compute projections and return as unordered_map -- user should implement - return std::unordered_map>{}; + // compute projections and return as unordered_map -- user should implement + return std::unordered_map>{}; } template auto RadhydroSimulation::ComputeStatistics() -> std::map { - // compute statistics and return a std::map -- user should implement - // IMPORTANT: the user is responsible for performing any necessary MPI reductions before returning - return std::map{}; + // compute statistics and return a std::map -- user should implement + // IMPORTANT: the user is responsible for performing any necessary MPI reductions before returning + return std::map{}; } template void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) { - // tag cells for refinement -- user should implement + // tag cells for refinement -- user should implement } template void RadhydroSimulation::computeReferenceSolution(amrex::MultiFab &ref, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo) + amrex::GpuArray const &prob_lo) { - // user should implement + // user should implement } template void RadhydroSimulation::computeAfterEvolve(amrex::Vector &initSumCons) { - amrex::GpuArray const &dx0 = geom[0].CellSizeArray(); - amrex::Real const vol = AMREX_D_TERM(dx0[0], *dx0[1], *dx0[2]); - - // check conservation of total energy - amrex::Real const Egas0 = initSumCons[RadSystem::gasEnergy_index]; - amrex::Real const Egas = state_new_cc_[0].sum(RadSystem::gasEnergy_index) * vol; - - amrex::Real Etot0 = NAN; - amrex::Real Etot = NAN; - if constexpr (Physics_Traits::is_radiation_enabled) { - amrex::Real Erad0 = 0.; - for (int g = 0; g < Physics_Traits::nGroups; ++g) { - Erad0 += initSumCons[RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g]; - } - Etot0 = Egas0 + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad0; - amrex::Real Erad = 0.; - for (int g = 0; g < Physics_Traits::nGroups; ++g) { - Erad += state_new_cc_[0].sum(RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g) * vol; - } - Etot = Egas + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad; - } else { - Etot0 = Egas0; - Etot = Egas; - } - - amrex::Real const abs_err = (Etot - Etot0); - amrex::Real const rel_err = abs_err / Etot0; - - amrex::Print() << "\nInitial gas+radiation energy = " << Etot0 << std::endl; - amrex::Print() << "Final gas+radiation energy = " << Etot << std::endl; - amrex::Print() << "\tabsolute conservation error = " << abs_err << std::endl; - amrex::Print() << "\trelative conservation error = " << rel_err << std::endl; - amrex::Print() << std::endl; - - if (computeReferenceSolution_) { - // compute reference solution - const int ncomp = state_new_cc_[0].nComp(); - const int nghost = state_new_cc_[0].nGrow(); - amrex::MultiFab state_ref_level0(boxArray(0), DistributionMap(0), ncomp, nghost); - computeReferenceSolution(state_ref_level0, geom[0].CellSizeArray(), geom[0].ProbLoArray()); - - // compute error norm - amrex::MultiFab residual(boxArray(0), DistributionMap(0), ncomp, nghost); - amrex::MultiFab::Copy(residual, state_ref_level0, 0, 0, ncomp, nghost); - amrex::MultiFab::Saxpy(residual, -1., state_new_cc_[0], 0, 0, ncomp, nghost); - - amrex::Real sol_norm = 0.; - amrex::Real err_norm = 0.; - // compute rms of each component - for (int n = 0; n < ncomp; ++n) { - sol_norm += std::pow(state_ref_level0.norm1(n), 2); - err_norm += std::pow(residual.norm1(n), 2); - } - sol_norm = std::sqrt(sol_norm); - err_norm = std::sqrt(err_norm); - - const double rel_error = err_norm / sol_norm; - errorNorm_ = rel_error; - amrex::Print() << "Relative rms L1 error norm = " << rel_error << std::endl; - } - amrex::Print() << std::endl; - - // compute average number of radiation subcycles per timestep - double const avg_rad_subcycles = static_cast(radiationCellUpdates_) / static_cast(cellUpdates_); - amrex::Print() << "avg. num. of radiation subcycles = " << avg_rad_subcycles << std::endl; - amrex::Print() << std::endl; + amrex::GpuArray const &dx0 = geom[0].CellSizeArray(); + amrex::Real const vol = AMREX_D_TERM(dx0[0], *dx0[1], *dx0[2]); + + // check conservation of total energy + amrex::Real const Egas0 = initSumCons[RadSystem::gasEnergy_index]; + amrex::Real const Egas = state_new_cc_[0].sum(RadSystem::gasEnergy_index) * vol; + + amrex::Real Etot0 = NAN; + amrex::Real Etot = NAN; + if constexpr (Physics_Traits::is_radiation_enabled) { + amrex::Real Erad0 = 0.; + for (int g = 0; g < Physics_Traits::nGroups; ++g) { + Erad0 += initSumCons[RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g]; + } + Etot0 = Egas0 + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad0; + amrex::Real Erad = 0.; + for (int g = 0; g < Physics_Traits::nGroups; ++g) { + Erad += state_new_cc_[0].sum(RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g) * vol; + } + Etot = Egas + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad; + } else { + Etot0 = Egas0; + Etot = Egas; + } + + amrex::Real const abs_err = (Etot - Etot0); + amrex::Real const rel_err = abs_err / Etot0; + + amrex::Print() << "\nInitial gas+radiation energy = " << Etot0 << std::endl; + amrex::Print() << "Final gas+radiation energy = " << Etot << std::endl; + amrex::Print() << "\tabsolute conservation error = " << abs_err << std::endl; + amrex::Print() << "\trelative conservation error = " << rel_err << std::endl; + amrex::Print() << std::endl; + + if (computeReferenceSolution_) { + // compute reference solution + const int ncomp = state_new_cc_[0].nComp(); + const int nghost = state_new_cc_[0].nGrow(); + amrex::MultiFab state_ref_level0(boxArray(0), DistributionMap(0), ncomp, nghost); + computeReferenceSolution(state_ref_level0, geom[0].CellSizeArray(), geom[0].ProbLoArray()); + + // compute error norm + amrex::MultiFab residual(boxArray(0), DistributionMap(0), ncomp, nghost); + amrex::MultiFab::Copy(residual, state_ref_level0, 0, 0, ncomp, nghost); + amrex::MultiFab::Saxpy(residual, -1., state_new_cc_[0], 0, 0, ncomp, nghost); + + amrex::Real sol_norm = 0.; + amrex::Real err_norm = 0.; + // compute rms of each component + for (int n = 0; n < ncomp; ++n) { + sol_norm += std::pow(state_ref_level0.norm1(n), 2); + err_norm += std::pow(residual.norm1(n), 2); + } + sol_norm = std::sqrt(sol_norm); + err_norm = std::sqrt(err_norm); + + const double rel_error = err_norm / sol_norm; + errorNorm_ = rel_error; + amrex::Print() << "Relative rms L1 error norm = " << rel_error << std::endl; + } + amrex::Print() << std::endl; + + // compute average number of radiation subcycles per timestep + double const avg_rad_subcycles = static_cast(radiationCellUpdates_) / static_cast(cellUpdates_); + amrex::Print() << "avg. num. of radiation subcycles = " << avg_rad_subcycles << std::endl; + amrex::Print() << std::endl; } template void RadhydroSimulation::advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) { - BL_PROFILE("RadhydroSimulation::advanceSingleTimestepAtLevel()"); - - // get flux registers - amrex::YAFluxRegister *fr_as_crse = nullptr; - amrex::YAFluxRegister *fr_as_fine = nullptr; - if (do_reflux != 0) { - if (lev < finestLevel()) { - fr_as_crse = flux_reg_[lev + 1].get(); - fr_as_crse->reset(); - } - if (lev > 0) { - fr_as_fine = flux_reg_[lev].get(); - } - } - - // since we are starting a new timestep, need to swap old and new state vectors - std::swap(state_old_cc_[lev], state_new_cc_[lev]); - if (Physics_Indices::nvarTotal_fc > 0) { - std::swap(state_old_fc_[lev], state_new_fc_[lev]); - } - - // check hydro states before update (this can be caused by the flux register!) - CHECK_HYDRO_STATES(state_old_cc_[lev]); - - // advance hydro - if constexpr (Physics_Traits::is_hydro_enabled) { - advanceHydroAtLevelWithRetries(lev, time, dt_lev, fr_as_crse, fr_as_fine); - } else { - // copy hydro vars from state_old_cc_ to state_new_cc_ - // (otherwise radiation update will be wrong!) - amrex::MultiFab::Copy(state_new_cc_[lev], state_old_cc_[lev], 0, 0, ncompHydro_, 0); - } - - // check hydro states after hydro update - CHECK_HYDRO_STATES(state_new_cc_[lev]); - - // subcycle radiation - if constexpr (Physics_Traits::is_radiation_enabled) { - subcycleRadiationAtLevel(lev, time, dt_lev, fr_as_crse, fr_as_fine); - } - - // check hydro states after radiation update - CHECK_HYDRO_STATES(state_new_cc_[lev]); - - // compute any operator-split terms here (user-defined) - computeAfterLevelAdvance(lev, time, dt_lev, ncycle); - - // check hydro states after user work - CHECK_HYDRO_STATES(state_new_cc_[lev]); - - // check state validity - AMREX_ASSERT(!state_new_cc_[lev].contains_nan(0, state_new_cc_[lev].nComp())); + BL_PROFILE("RadhydroSimulation::advanceSingleTimestepAtLevel()"); + + // get flux registers + amrex::YAFluxRegister *fr_as_crse = nullptr; + amrex::YAFluxRegister *fr_as_fine = nullptr; + if (do_reflux != 0) { + if (lev < finestLevel()) { + fr_as_crse = flux_reg_[lev + 1].get(); + fr_as_crse->reset(); + } + if (lev > 0) { + fr_as_fine = flux_reg_[lev].get(); + } + } + + // since we are starting a new timestep, need to swap old and new state vectors + std::swap(state_old_cc_[lev], state_new_cc_[lev]); + if (Physics_Indices::nvarTotal_fc > 0) { + std::swap(state_old_fc_[lev], state_new_fc_[lev]); + } + + // check hydro states before update (this can be caused by the flux register!) + CHECK_HYDRO_STATES(state_old_cc_[lev]); + + // advance hydro + if constexpr (Physics_Traits::is_hydro_enabled) { + advanceHydroAtLevelWithRetries(lev, time, dt_lev, fr_as_crse, fr_as_fine); + } else { + // copy hydro vars from state_old_cc_ to state_new_cc_ + // (otherwise radiation update will be wrong!) + amrex::MultiFab::Copy(state_new_cc_[lev], state_old_cc_[lev], 0, 0, ncompHydro_, 0); + } + + // check hydro states after hydro update + CHECK_HYDRO_STATES(state_new_cc_[lev]); + + // subcycle radiation + if constexpr (Physics_Traits::is_radiation_enabled) { + subcycleRadiationAtLevel(lev, time, dt_lev, fr_as_crse, fr_as_fine); + } + + // check hydro states after radiation update + CHECK_HYDRO_STATES(state_new_cc_[lev]); + + // compute any operator-split terms here (user-defined) + computeAfterLevelAdvance(lev, time, dt_lev, ncycle); + + // check hydro states after user work + CHECK_HYDRO_STATES(state_new_cc_[lev]); + + // check state validity + AMREX_ASSERT(!state_new_cc_[lev].contains_nan(0, state_new_cc_[lev].nComp())); } template void RadhydroSimulation::fillPoissonRhsAtLevel(amrex::MultiFab &rhs_mf, const int lev) { - // add hydro density to Poisson rhs - auto const &state = state_new_cc_[lev].const_arrays(); - auto rhs = rhs_mf.arrays(); - const Real G = Gconst_; - - amrex::ParallelFor(rhs_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - // *add* density to rhs_mf - // (N.B. particles **will not work** if you overwrite the density here!) - rhs[bx](i, j, k) += 4.0 * M_PI * G * state[bx](i, j, k, HydroSystem::density_index); - }); - amrex::Gpu::streamSynchronizeAll(); + // add hydro density to Poisson rhs + auto const &state = state_new_cc_[lev].const_arrays(); + auto rhs = rhs_mf.arrays(); + const Real G = Gconst_; + + amrex::ParallelFor(rhs_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + // *add* density to rhs_mf + // (N.B. particles **will not work** if you overwrite the density here!) + rhs[bx](i, j, k) += 4.0 * M_PI * G * state[bx](i, j, k, HydroSystem::density_index); + }); + amrex::Gpu::streamSynchronizeAll(); } template void RadhydroSimulation::applyPoissonGravityAtLevel(amrex::MultiFab const &phi_mf, const int lev, const amrex::Real dt) { - if constexpr (AMREX_SPACEDIM == 3) { - // apply Poisson gravity operator on level 'lev' - auto const &dx = geom[lev].CellSizeArray(); - auto const &phi = phi_mf.const_arrays(); - auto state = state_new_cc_[lev].arrays(); - - amrex::ParallelFor(phi_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - // add operator-split gravitational acceleration - const amrex::Real rho = state[bx](i, j, k, HydroSystem::density_index); - amrex::Real px = state[bx](i, j, k, HydroSystem::x1Momentum_index); - amrex::Real py = state[bx](i, j, k, HydroSystem::x2Momentum_index); - amrex::Real pz = state[bx](i, j, k, HydroSystem::x3Momentum_index); - const amrex::Real KE_old = 0.5 * (px * px + py * py + pz * pz) / rho; - - // g = -grad \phi - amrex::Real gx = -0.5 * (phi[bx](i + 1, j, k) - phi[bx](i - 1, j, k)) / dx[0]; - amrex::Real gy = -0.5 * (phi[bx](i, j + 1, k) - phi[bx](i, j - 1, k)) / dx[1]; - amrex::Real gz = -0.5 * (phi[bx](i, j, k + 1) - phi[bx](i, j, k - 1)) / dx[2]; - - px += dt * rho * gx; - py += dt * rho * gy; - pz += dt * rho * gz; - const amrex::Real KE_new = 0.5 * (px * px + py * py + pz * pz) / rho; - const amrex::Real dKE = KE_new - KE_old; - - state[bx](i, j, k, HydroSystem::x1Momentum_index) = px; - state[bx](i, j, k, HydroSystem::x2Momentum_index) = py; - state[bx](i, j, k, HydroSystem::x3Momentum_index) = pz; - state[bx](i, j, k, HydroSystem::energy_index) += dKE; - }); - } + if constexpr (AMREX_SPACEDIM == 3) { + // apply Poisson gravity operator on level 'lev' + auto const &dx = geom[lev].CellSizeArray(); + auto const &phi = phi_mf.const_arrays(); + auto state = state_new_cc_[lev].arrays(); + + amrex::ParallelFor(phi_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + // add operator-split gravitational acceleration + const amrex::Real rho = state[bx](i, j, k, HydroSystem::density_index); + amrex::Real px = state[bx](i, j, k, HydroSystem::x1Momentum_index); + amrex::Real py = state[bx](i, j, k, HydroSystem::x2Momentum_index); + amrex::Real pz = state[bx](i, j, k, HydroSystem::x3Momentum_index); + const amrex::Real KE_old = 0.5 * (px * px + py * py + pz * pz) / rho; + + // g = -grad \phi + amrex::Real gx = -0.5 * (phi[bx](i + 1, j, k) - phi[bx](i - 1, j, k)) / dx[0]; + amrex::Real gy = -0.5 * (phi[bx](i, j + 1, k) - phi[bx](i, j - 1, k)) / dx[1]; + amrex::Real gz = -0.5 * (phi[bx](i, j, k + 1) - phi[bx](i, j, k - 1)) / dx[2]; + + px += dt * rho * gx; + py += dt * rho * gy; + pz += dt * rho * gz; + const amrex::Real KE_new = 0.5 * (px * px + py * py + pz * pz) / rho; + const amrex::Real dKE = KE_new - KE_old; + + state[bx](i, j, k, HydroSystem::x1Momentum_index) = px; + state[bx](i, j, k, HydroSystem::x2Momentum_index) = py; + state[bx](i, j, k, HydroSystem::x3Momentum_index) = pz; + state[bx](i, j, k, HydroSystem::energy_index) += dKE; + }); + } } // fix-up any unphysical states created by AMR operations // (e.g., caused by the flux register or from interpolation) template void RadhydroSimulation::FixupState(int lev) { - BL_PROFILE("RadhydroSimulation::FixupState()"); + BL_PROFILE("RadhydroSimulation::FixupState()"); - // fix hydro state - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); - // sync internal energy and total energy - HydroSystem::SyncDualEnergy(state_new_cc_[lev]); + // fix hydro state + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); + // sync internal energy and total energy + HydroSystem::SyncDualEnergy(state_new_cc_[lev]); } // Compute a new multifab 'mf' by copying in state from valid region and filling @@ -749,1110 +752,1110 @@ template void RadhydroSimulation::FixupState(int // are implemented in this class (and must be *static* functions). template void RadhydroSimulation::FillPatch(int lev, amrex::Real time, amrex::MultiFab &mf, int icomp, int ncomp, quokka::centering cen, - quokka::direction dir, FillPatchType fptype) + quokka::direction dir, FillPatchType fptype) { - BL_PROFILE("AMRSimulation::FillPatch()"); - - amrex::Vector cmf; - amrex::Vector fmf; - amrex::Vector ctime; - amrex::Vector ftime; - - if (lev == 0) { - // in this case, should return either state_new_[lev] or state_old_[lev] - GetData(lev, time, fmf, ftime, cen, dir); - } else { - // in this case, should return either state_new_[lev] or state_old_[lev] - // returns old state, new state, or both depending on 'time' - GetData(lev, time, fmf, ftime, cen, dir); - GetData(lev - 1, time, cmf, ctime, cen, dir); - } - - if (cen == quokka::centering::cc) { - FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_cc_, cen, fptype, PreInterpState, PostInterpState); - } else if (cen == quokka::centering::fc) { - FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_fc_, cen, fptype, PreInterpState, PostInterpState); - } + BL_PROFILE("AMRSimulation::FillPatch()"); + + amrex::Vector cmf; + amrex::Vector fmf; + amrex::Vector ctime; + amrex::Vector ftime; + + if (lev == 0) { + // in this case, should return either state_new_[lev] or state_old_[lev] + GetData(lev, time, fmf, ftime, cen, dir); + } else { + // in this case, should return either state_new_[lev] or state_old_[lev] + // returns old state, new state, or both depending on 'time' + GetData(lev, time, fmf, ftime, cen, dir); + GetData(lev - 1, time, cmf, ctime, cen, dir); + } + + if (cen == quokka::centering::cc) { + FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_cc_, cen, fptype, PreInterpState, PostInterpState); + } else if (cen == quokka::centering::fc) { + FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_fc_, cen, fptype, PreInterpState, PostInterpState); + } } template void RadhydroSimulation::PreInterpState(amrex::MultiFab &mf, int scomp, int ncomp) { - BL_PROFILE("RadhydroSimulation::PreInterpState()"); - - auto const &cons = mf.arrays(); - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { - const auto rho = cons[bx](i, j, k, HydroSystem::density_index); - const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); - const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); - const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); - const auto Etot = cons[bx](i, j, k, HydroSystem::energy_index); - const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); - - // replace hydro total energy with specific internal energy (SIE) - const auto e = (Etot - kinetic_energy) / rho; - cons[bx](i, j, k, HydroSystem::energy_index) = e; - }); + BL_PROFILE("RadhydroSimulation::PreInterpState()"); + + auto const &cons = mf.arrays(); + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { + const auto rho = cons[bx](i, j, k, HydroSystem::density_index); + const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); + const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); + const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); + const auto Etot = cons[bx](i, j, k, HydroSystem::energy_index); + const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); + + // replace hydro total energy with specific internal energy (SIE) + const auto e = (Etot - kinetic_energy) / rho; + cons[bx](i, j, k, HydroSystem::energy_index) = e; + }); } template void RadhydroSimulation::PostInterpState(amrex::MultiFab &mf, int scomp, int ncomp) { - BL_PROFILE("RadhydroSimulation::PostInterpState()"); - - auto const &cons = mf.arrays(); - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { - const auto rho = cons[bx](i, j, k, HydroSystem::density_index); - const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); - const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); - const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); - const auto e = cons[bx](i, j, k, HydroSystem::energy_index); - const auto Eint = rho * e; - const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); - - // recompute hydro total energy from Eint + KE - const auto Etot = Eint + kinetic_energy; - cons[bx](i, j, k, HydroSystem::energy_index) = Etot; - }); + BL_PROFILE("RadhydroSimulation::PostInterpState()"); + + auto const &cons = mf.arrays(); + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { + const auto rho = cons[bx](i, j, k, HydroSystem::density_index); + const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); + const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); + const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); + const auto e = cons[bx](i, j, k, HydroSystem::energy_index); + const auto Eint = rho * e; + const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); + + // recompute hydro total energy from Eint + KE + const auto Etot = Eint + kinetic_energy; + cons[bx](i, j, k, HydroSystem::energy_index) = Etot; + }); } template template auto RadhydroSimulation::computeAxisAlignedProfile(const int axis, F const &user_f) -> amrex::Gpu::HostVector { - // compute a 1D profile of user_f(i, j, k, state) along the given axis. - BL_PROFILE("RadhydroSimulation::computeAxisAlignedProfile()"); - - // allocate temporary multifabs - amrex::Vector q; - q.resize(finest_level + 1); - for (int lev = 0; lev <= finest_level; ++lev) { - q[lev].define(boxArray(lev), DistributionMap(lev), 1, 0); - } - - // evaluate user_f on all levels - for (int lev = 0; lev <= finest_level; ++lev) { - for (amrex::MFIter iter(q[lev]); iter.isValid(); ++iter) { - auto const &box = iter.validbox(); - auto const &state = state_new_cc_[lev].const_array(iter); - auto const &result = q[lev].array(iter); - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { result(i, j, k) = user_f(i, j, k, state); }); - } - } - - // average down - for (int crse_lev = finest_level - 1; crse_lev >= 0; --crse_lev) { - amrex::average_down(q[crse_lev + 1], q[crse_lev], geom[crse_lev + 1], geom[crse_lev], 0, q[crse_lev].nComp(), refRatio(crse_lev)); - } - - // compute 1D profile from level 0 multifab - amrex::Box domain = geom[0].Domain(); - auto profile = amrex::sumToLine(q[0], 0, q[0].nComp(), domain, axis); - - // normalize profile - amrex::Long numCells = domain.numPts() / domain.length(axis); - for (double &bin : profile) { - bin /= static_cast(numCells); - } - - return profile; + // compute a 1D profile of user_f(i, j, k, state) along the given axis. + BL_PROFILE("RadhydroSimulation::computeAxisAlignedProfile()"); + + // allocate temporary multifabs + amrex::Vector q; + q.resize(finest_level + 1); + for (int lev = 0; lev <= finest_level; ++lev) { + q[lev].define(boxArray(lev), DistributionMap(lev), 1, 0); + } + + // evaluate user_f on all levels + for (int lev = 0; lev <= finest_level; ++lev) { + for (amrex::MFIter iter(q[lev]); iter.isValid(); ++iter) { + auto const &box = iter.validbox(); + auto const &state = state_new_cc_[lev].const_array(iter); + auto const &result = q[lev].array(iter); + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { result(i, j, k) = user_f(i, j, k, state); }); + } + } + + // average down + for (int crse_lev = finest_level - 1; crse_lev >= 0; --crse_lev) { + amrex::average_down(q[crse_lev + 1], q[crse_lev], geom[crse_lev + 1], geom[crse_lev], 0, q[crse_lev].nComp(), refRatio(crse_lev)); + } + + // compute 1D profile from level 0 multifab + amrex::Box domain = geom[0].Domain(); + auto profile = amrex::sumToLine(q[0], 0, q[0].nComp(), domain, axis); + + // normalize profile + amrex::Long numCells = domain.numPts() / domain.length(axis); + for (double &bin : profile) { + bin /= static_cast(numCells); + } + + return profile; } template void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine) + amrex::YAFluxRegister *fr_as_fine) { - BL_PROFILE_REGION("HydroSolver"); - // timestep retries - const int max_retries = 4; - bool success = false; - - // save the pre-advance fine flux register state in originalFineData - amrex::MultiFab originalFineData; - if (fr_as_fine != nullptr) { - amrex::MultiFab const &fineData = fr_as_fine->getFineData(); - originalFineData.define(fineData.boxArray(), fineData.DistributionMap(), fineData.nComp(), 0); - amrex::Copy(originalFineData, fineData, 0, 0, fineData.nComp(), 0); - } + BL_PROFILE_REGION("HydroSolver"); + // timestep retries + const int max_retries = 4; + bool success = false; + + // save the pre-advance fine flux register state in originalFineData + amrex::MultiFab originalFineData; + if (fr_as_fine != nullptr) { + amrex::MultiFab const &fineData = fr_as_fine->getFineData(); + originalFineData.define(fineData.boxArray(), fineData.DistributionMap(), fineData.nComp(), 0); + amrex::Copy(originalFineData, fineData, 0, 0, fineData.nComp(), 0); + } #ifdef AMREX_PARTICLES - amrex::AmrTracerParticleContainer::ContainerLike originalTracerPC; - if (do_tracers != 0) { - // save the pre-advance tracer particles - originalTracerPC = TracerPC->make_alike(); - } + amrex::AmrTracerParticleContainer::ContainerLike originalTracerPC; + if (do_tracers != 0) { + // save the pre-advance tracer particles + originalTracerPC = TracerPC->make_alike(); + } #endif - for (int retry_count = 0; retry_count <= max_retries; ++retry_count) { - // reduce timestep by a factor of 2^retry_count - const int nsubsteps = std::pow(2, retry_count); - const amrex::Real dt_step = dt_lev / nsubsteps; - - if (retry_count > 0 && Verbose()) { - amrex::Print() << "\t>> Re-trying hydro advance at level " << lev << " with reduced timestep (nsubsteps = " << nsubsteps - << ", dt_new = " << dt_step << ")\n"; - } - - if (retry_count > 0) { - // reset the flux registers to their pre-advance state - if (fr_as_crse != nullptr) { - fr_as_crse->reset(); - } - if (fr_as_fine != nullptr) { - amrex::Copy(fr_as_fine->getFineData(), originalFineData, 0, 0, originalFineData.nComp(), 0); - } + for (int retry_count = 0; retry_count <= max_retries; ++retry_count) { + // reduce timestep by a factor of 2^retry_count + const int nsubsteps = std::pow(2, retry_count); + const amrex::Real dt_step = dt_lev / nsubsteps; + + if (retry_count > 0 && Verbose()) { + amrex::Print() << "\t>> Re-trying hydro advance at level " << lev << " with reduced timestep (nsubsteps = " << nsubsteps + << ", dt_new = " << dt_step << ")\n"; + } + + if (retry_count > 0) { + // reset the flux registers to their pre-advance state + if (fr_as_crse != nullptr) { + fr_as_crse->reset(); + } + if (fr_as_fine != nullptr) { + amrex::Copy(fr_as_fine->getFineData(), originalFineData, 0, 0, originalFineData.nComp(), 0); + } #ifdef AMREX_PARTICLES - if (do_tracers != 0) { - // reset the tracer particles to their pre-advance state - TracerPC->copyParticles(originalTracerPC, true); - } + if (do_tracers != 0) { + // reset the tracer particles to their pre-advance state + TracerPC->copyParticles(originalTracerPC, true); + } #endif - } - - // create temporary multifab for old state - amrex::MultiFab state_old_cc_tmp(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); - amrex::Copy(state_old_cc_tmp, state_old_cc_[lev], 0, 0, Physics_Indices::nvarTotal_cc, nghost_cc_); - - // subcycle advanceHydroAtLevel, checking return value - for (int substep = 0; substep < nsubsteps; ++substep) { - if (substep > 0) { - // since we are starting a new substep, we need to copy hydro state from - // the new state vector to old state vector - amrex::Copy(state_old_cc_tmp, state_new_cc_[lev], 0, 0, ncompHydro_, nghost_cc_); - } - - success = advanceHydroAtLevel(state_old_cc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); - - if (!success) { - if (Verbose()) { - amrex::Print() << "\t>> WARNING: Hydro advance failed on level " << lev << "\n"; - } - break; - } - } - - if (success) { - // we are done, do not attempt more retries - break; - } - } - - if (!success) { - // crash, we have exceeded max_retries - amrex::Print() << "\nQUOKKA FATAL ERROR\n" - << "Hydro update exceeded max_retries on level " << lev << ". Cannot continue, crashing...\n" - << std::endl; - - // write plotfile or Ascent Blueprint file - amrex::ParallelDescriptor::Barrier(); + } + + // create temporary multifab for old state + amrex::MultiFab state_old_cc_tmp(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); + amrex::Copy(state_old_cc_tmp, state_old_cc_[lev], 0, 0, Physics_Indices::nvarTotal_cc, nghost_cc_); + + // subcycle advanceHydroAtLevel, checking return value + for (int substep = 0; substep < nsubsteps; ++substep) { + if (substep > 0) { + // since we are starting a new substep, we need to copy hydro state from + // the new state vector to old state vector + amrex::Copy(state_old_cc_tmp, state_new_cc_[lev], 0, 0, ncompHydro_, nghost_cc_); + } + + success = advanceHydroAtLevel(state_old_cc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); + + if (!success) { + if (Verbose()) { + amrex::Print() << "\t>> WARNING: Hydro advance failed on level " << lev << "\n"; + } + break; + } + } + + if (success) { + // we are done, do not attempt more retries + break; + } + } + + if (!success) { + // crash, we have exceeded max_retries + amrex::Print() << "\nQUOKKA FATAL ERROR\n" + << "Hydro update exceeded max_retries on level " << lev << ". Cannot continue, crashing...\n" + << std::endl; + + // write plotfile or Ascent Blueprint file + amrex::ParallelDescriptor::Barrier(); #ifdef AMREX_USE_ASCENT - conduit::Node mesh; - amrex::SingleLevelToBlueprint(state_new_cc_[lev], componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); - conduit::Node bpMeshHost; - bpMeshHost.set(mesh); // copy to host mem (needed for Blueprint HDF5 output) - amrex::WriteBlueprintFiles(bpMeshHost, "debug_hydro_state_fatal", istep[lev] + 1, "hdf5"); + conduit::Node mesh; + amrex::SingleLevelToBlueprint(state_new_cc_[lev], componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); + conduit::Node bpMeshHost; + bpMeshHost.set(mesh); // copy to host mem (needed for Blueprint HDF5 output) + amrex::WriteBlueprintFiles(bpMeshHost, "debug_hydro_state_fatal", istep[lev] + 1, "hdf5"); #else - WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], time, - istep[lev] + 1); + WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], time, + istep[lev] + 1); #endif - amrex::ParallelDescriptor::Barrier(); + amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::ParallelDescriptor::Abort(); - } - } + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } + } } template auto RadhydroSimulation::isCflViolated(int lev, amrex::Real time, amrex::Real dt_actual) -> bool { - // check whether dt_actual would violate CFL condition using the post-update hydro state - - // compute max signal speed - amrex::Real max_signal = HydroSystem::maxSignalSpeedLocal(state_new_cc_[lev]); - amrex::ParallelDescriptor::ReduceRealMax(max_signal); - - // compute dt_cfl - auto dx = geom[lev].CellSizeArray(); - const amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); - const amrex::Real dt_cfl = cflNumber_ * (dx_min / max_signal); - - // check whether dt_actual > dt_cfl (CFL violation) - const amrex::Real max_factor = 1.1; - const bool cflViolation = dt_actual > (max_factor * dt_cfl); - if (cflViolation && Verbose()) { - amrex::Print() << "\t>> CFL violation detected on level " << lev << " with dt_lev = " << dt_actual << " and dt_cfl = " << dt_cfl << "\n" - << "\t max_signal = " << max_signal << "\n"; - } - return cflViolation; + // check whether dt_actual would violate CFL condition using the post-update hydro state + + // compute max signal speed + amrex::Real max_signal = HydroSystem::maxSignalSpeedLocal(state_new_cc_[lev]); + amrex::ParallelDescriptor::ReduceRealMax(max_signal); + + // compute dt_cfl + auto dx = geom[lev].CellSizeArray(); + const amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); + const amrex::Real dt_cfl = cflNumber_ * (dx_min / max_signal); + + // check whether dt_actual > dt_cfl (CFL violation) + const amrex::Real max_factor = 1.1; + const bool cflViolation = dt_actual > (max_factor * dt_cfl); + if (cflViolation && Verbose()) { + amrex::Print() << "\t>> CFL violation detected on level " << lev << " with dt_lev = " << dt_actual << " and dt_cfl = " << dt_cfl << "\n" + << "\t max_signal = " << max_signal << "\n"; + } + return cflViolation; } template void RadhydroSimulation::printCoordinates(int lev, const amrex::IntVect &cell_idx) { - amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; - if (AMREX_SPACEDIM > 1) { - y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + if (AMREX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMREX_SPACEDIM > 2) { - z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - } - } - amrex::Print() << "Coordinates: (" << x_coord << ", " << y_coord << ", " << z_coord << "): "; + if (AMREX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } + amrex::Print() << "Coordinates: (" << x_coord << ", " << y_coord << ", " << z_coord << "): "; } template auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, - int lev, amrex::Real time, amrex::Real dt_lev) -> bool + int lev, amrex::Real time, amrex::Real dt_lev) -> bool { - BL_PROFILE("RadhydroSimulation::advanceHydroAtLevel()"); - - amrex::Real fluxScaleFactor = NAN; - if (integratorOrder_ == 2) { - fluxScaleFactor = 0.5; - } else if (integratorOrder_ == 1) { - fluxScaleFactor = 1.0; - } - - auto dx = geom[lev].CellSizeArray(); - - // do Strang split source terms (first half-step) - addStrangSplitSourcesWithBuiltin(state_old_cc_tmp, lev, time, 0.5 * dt_lev); - - // create temporary multifab for intermediate state - amrex::MultiFab state_inter_cc_(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); - state_inter_cc_.setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled - - // create temporary multifabs for combined RK2 flux and time-average face velocity - std::array flux_rk2; - std::array avgFaceVel; - const int nghost_vel = 2; // 2 ghost faces are needed for tracer particles - auto ba = grids[lev]; - auto dm = dmap[lev]; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - // initialize flux MultiFab - flux_rk2[idim] = amrex::MultiFab(ba_face, dm, ncompHydro_, 0); - flux_rk2[idim].setVal(0); - // initialize velocity MultiFab - avgFaceVel[idim] = amrex::MultiFab(ba_face, dm, 1, nghost_vel); - avgFaceVel[idim].setVal(0); - } - - // update ghost zones [old timestep] - fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); - - // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) - if (lowLevelDebuggingOutput_ == 1) { + BL_PROFILE("RadhydroSimulation::advanceHydroAtLevel()"); + + amrex::Real fluxScaleFactor = NAN; + if (integratorOrder_ == 2) { + fluxScaleFactor = 0.5; + } else if (integratorOrder_ == 1) { + fluxScaleFactor = 1.0; + } + + auto dx = geom[lev].CellSizeArray(); + + // do Strang split source terms (first half-step) + addStrangSplitSourcesWithBuiltin(state_old_cc_tmp, lev, time, 0.5 * dt_lev); + + // create temporary multifab for intermediate state + amrex::MultiFab state_inter_cc_(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); + state_inter_cc_.setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled + + // create temporary multifabs for combined RK2 flux and time-average face velocity + std::array flux_rk2; + std::array avgFaceVel; + const int nghost_vel = 2; // 2 ghost faces are needed for tracer particles + auto ba = grids[lev]; + auto dm = dmap[lev]; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + // initialize flux MultiFab + flux_rk2[idim] = amrex::MultiFab(ba_face, dm, ncompHydro_, 0); + flux_rk2[idim].setVal(0); + // initialize velocity MultiFab + avgFaceVel[idim] = amrex::MultiFab(ba_face, dm, 1, nghost_vel); + avgFaceVel[idim].setVal(0); + } + + // update ghost zones [old timestep] + fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); + + // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) + if (lowLevelDebuggingOutput_ == 1) { #ifdef AMREX_USE_ASCENT - // write Blueprint HDF5 files - conduit::Node mesh; - amrex::SingleLevelToBlueprint(state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); - amrex::WriteBlueprintFiles(mesh, "debug_stage1_filled_state_old", istep[lev] + 1, "hdf5"); + // write Blueprint HDF5 files + conduit::Node mesh; + amrex::SingleLevelToBlueprint(state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); + amrex::WriteBlueprintFiles(mesh, "debug_stage1_filled_state_old", istep[lev] + 1, "hdf5"); #else - // write AMReX plotfile - // WriteSingleLevelPlotfile(CustomPlotFileName("debug_stage1_filled_state_old", istep[lev]+1), - // state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev]+1); + // write AMReX plotfile + // WriteSingleLevelPlotfile(CustomPlotFileName("debug_stage1_filled_state_old", istep[lev]+1), + // state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev]+1); #endif - } - - // check state validity - AMREX_ASSERT(!state_old_cc_tmp.contains_nan(0, state_old_cc_tmp.nComp())); - AMREX_ASSERT(!state_old_cc_tmp.contains_nan()); // check ghost cells - - auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); - - // Stage 1 of RK2-SSP - { - // advance all grids on local processor (Stage 1 of integrator) - auto const &stateOld = state_old_cc_tmp; - auto &stateNew = state_inter_cc_; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); - amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); - } - - amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); - amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); - redoFlag.setVal(quokka::redoFlag::none); - - HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); - - // LOW LEVEL DEBUGGING: output rhs - if (lowLevelDebuggingOutput_ == 1) { - // write rhs - std::string plotfile_name = CustomPlotFileName("debug_stage1_rhs_fluxes", istep[lev] + 1); - WriteSingleLevelPlotfile(plotfile_name, rhs, componentNames_cc_, geom[lev], time, istep[lev] + 1); - - // write fluxes - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string fullprefix = - amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("Flux_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(fluxArrays[idim], fullprefix); - } - // write face velocities - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string fullprefix = amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", - std::string("FaceVel_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(faceVel[idim], fullprefix); - } - } - - // do first-order flux correction (FOFC) - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long const ncells_bad = redoFlag.sum(0); - if (ncells_bad > 0) { - if (Verbose()) { - amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - // Calculate the coordinates based on the cell index and cell size - printCoordinates(lev, cell_idx); - amrex::print_state(stateNew, cell_idx); - } - - // synchronize redoFlag across ranks - redoFlag.FillBoundary(geom[lev].periodicity()); - - // replace fluxes around troubled cells with Godunov fluxes - replaceFluxes(fluxArrays, FOfluxArrays, redoFlag); - replaceFluxes(faceVel, FOfaceVel, redoFlag); // needed for dual energy - - // re-do RK update - HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); - - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long const ncells_bad = static_cast(redoFlag.sum(0)); - if (ncells_bad > 0) { - // FOFC failed - if (Verbose()) { - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - // print cell state - amrex::Print() << "[FOFC-1] Flux correction failed:\n"; - printCoordinates(lev, cell_idx); - amrex::print_state(stateNew, cell_idx); - amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; - } - if (abortOnFofcFailure_ != 0) { - return false; - } - } - } - - // prevent vacuum - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); - - if (useDualEnergy_ == 1) { - // sync internal energy (requires positive density) - HydroSystem::SyncDualEnergy(stateNew); - } - - if (do_reflux == 1) { - // increment flux registers - incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); - } - } - amrex::Gpu::streamSynchronizeAll(); - - // Stage 2 of RK2-SSP - if (integratorOrder_ == 2) { - // update ghost zones [intermediate stage stored in state_inter_cc_] - fillBoundaryConditions(state_inter_cc_, state_inter_cc_, lev, time + dt_lev, quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // check intermediate state validity - AMREX_ASSERT(!state_inter_cc_.contains_nan(0, state_inter_cc_.nComp())); - AMREX_ASSERT(!state_inter_cc_.contains_nan()); // check ghost zones - - auto const &stateOld = state_old_cc_tmp; - auto const &stateInter = state_inter_cc_; - auto &stateFinal = state_new_cc_[lev]; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateInter, ncompHydro_, lev); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); - amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); - } - - amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); - amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); - redoFlag.setVal(quokka::redoFlag::none); - - HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); - - // do first-order flux correction (FOFC) - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long const ncells_bad = redoFlag.sum(0); - if (ncells_bad > 0) { - if (Verbose()) { - amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - printCoordinates(lev, cell_idx); - amrex::print_state(stateFinal, cell_idx); - } - - // synchronize redoFlag across ranks - redoFlag.FillBoundary(geom[lev].periodicity()); - - // replace fluxes around troubled cells with Godunov fluxes - replaceFluxes(flux_rk2, FOfluxArrays, redoFlag); - replaceFluxes(avgFaceVel, FOfaceVel, redoFlag); // needed for dual energy - - // re-do RK update - HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); - - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long ncells_bad = redoFlag.sum(0); - if (ncells_bad > 0) { - // FOFC failed - if (Verbose()) { - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - // print cell state - amrex::Print() << "[FOFC-2] Flux correction failed:\n"; - printCoordinates(lev, cell_idx); - amrex::print_state(stateFinal, cell_idx); - amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; - } - if (abortOnFofcFailure_ != 0) { - return false; - } - } - } - - // prevent vacuum - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); - - if (useDualEnergy_ == 1) { - // sync internal energy (requires positive density) - HydroSystem::SyncDualEnergy(stateFinal); - } - - if (do_reflux == 1) { - // increment flux registers - incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); - } - } else { // we are only doing forward Euler - amrex::Copy(state_new_cc_[lev], state_inter_cc_, 0, 0, ncompHydro_, 0); - } - amrex::Gpu::streamSynchronizeAll(); - - // advect tracer particles using avgFaceVel + } + + // check state validity + AMREX_ASSERT(!state_old_cc_tmp.contains_nan(0, state_old_cc_tmp.nComp())); + AMREX_ASSERT(!state_old_cc_tmp.contains_nan()); // check ghost cells + + auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); + + // Stage 1 of RK2-SSP + { + // advance all grids on local processor (Stage 1 of integrator) + auto const &stateOld = state_old_cc_tmp; + auto &stateNew = state_inter_cc_; + auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); + amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); + } + + amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); + amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); + redoFlag.setVal(quokka::redoFlag::none); + + HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); + + // LOW LEVEL DEBUGGING: output rhs + if (lowLevelDebuggingOutput_ == 1) { + // write rhs + std::string plotfile_name = CustomPlotFileName("debug_stage1_rhs_fluxes", istep[lev] + 1); + WriteSingleLevelPlotfile(plotfile_name, rhs, componentNames_cc_, geom[lev], time, istep[lev] + 1); + + // write fluxes + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string fullprefix = + amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("Flux_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(fluxArrays[idim], fullprefix); + } + // write face velocities + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string fullprefix = amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", + std::string("FaceVel_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(faceVel[idim], fullprefix); + } + } + + // do first-order flux correction (FOFC) + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long const ncells_bad = redoFlag.sum(0); + if (ncells_bad > 0) { + if (Verbose()) { + amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // Calculate the coordinates based on the cell index and cell size + printCoordinates(lev, cell_idx); + amrex::print_state(stateNew, cell_idx); + } + + // synchronize redoFlag across ranks + redoFlag.FillBoundary(geom[lev].periodicity()); + + // replace fluxes around troubled cells with Godunov fluxes + replaceFluxes(fluxArrays, FOfluxArrays, redoFlag); + replaceFluxes(faceVel, FOfaceVel, redoFlag); // needed for dual energy + + // re-do RK update + HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); + + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long const ncells_bad = static_cast(redoFlag.sum(0)); + if (ncells_bad > 0) { + // FOFC failed + if (Verbose()) { + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // print cell state + amrex::Print() << "[FOFC-1] Flux correction failed:\n"; + printCoordinates(lev, cell_idx); + amrex::print_state(stateNew, cell_idx); + amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; + } + if (abortOnFofcFailure_ != 0) { + return false; + } + } + } + + // prevent vacuum + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); + + if (useDualEnergy_ == 1) { + // sync internal energy (requires positive density) + HydroSystem::SyncDualEnergy(stateNew); + } + + if (do_reflux == 1) { + // increment flux registers + incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); + } + } + amrex::Gpu::streamSynchronizeAll(); + + // Stage 2 of RK2-SSP + if (integratorOrder_ == 2) { + // update ghost zones [intermediate stage stored in state_inter_cc_] + fillBoundaryConditions(state_inter_cc_, state_inter_cc_, lev, time + dt_lev, quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // check intermediate state validity + AMREX_ASSERT(!state_inter_cc_.contains_nan(0, state_inter_cc_.nComp())); + AMREX_ASSERT(!state_inter_cc_.contains_nan()); // check ghost zones + + auto const &stateOld = state_old_cc_tmp; + auto const &stateInter = state_inter_cc_; + auto &stateFinal = state_new_cc_[lev]; + auto [fluxArrays, faceVel] = computeHydroFluxes(stateInter, ncompHydro_, lev); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); + amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); + } + + amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); + amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); + redoFlag.setVal(quokka::redoFlag::none); + + HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + + // do first-order flux correction (FOFC) + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long const ncells_bad = redoFlag.sum(0); + if (ncells_bad > 0) { + if (Verbose()) { + amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + printCoordinates(lev, cell_idx); + amrex::print_state(stateFinal, cell_idx); + } + + // synchronize redoFlag across ranks + redoFlag.FillBoundary(geom[lev].periodicity()); + + // replace fluxes around troubled cells with Godunov fluxes + replaceFluxes(flux_rk2, FOfluxArrays, redoFlag); + replaceFluxes(avgFaceVel, FOfaceVel, redoFlag); // needed for dual energy + + // re-do RK update + HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long ncells_bad = redoFlag.sum(0); + if (ncells_bad > 0) { + // FOFC failed + if (Verbose()) { + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // print cell state + amrex::Print() << "[FOFC-2] Flux correction failed:\n"; + printCoordinates(lev, cell_idx); + amrex::print_state(stateFinal, cell_idx); + amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; + } + if (abortOnFofcFailure_ != 0) { + return false; + } + } + } + + // prevent vacuum + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); + + if (useDualEnergy_ == 1) { + // sync internal energy (requires positive density) + HydroSystem::SyncDualEnergy(stateFinal); + } + + if (do_reflux == 1) { + // increment flux registers + incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); + } + } else { // we are only doing forward Euler + amrex::Copy(state_new_cc_[lev], state_inter_cc_, 0, 0, ncompHydro_, 0); + } + amrex::Gpu::streamSynchronizeAll(); + + // advect tracer particles using avgFaceVel #ifdef AMREX_PARTICLES - if (do_tracers != 0) { - // copy avgFaceVel to state_new_fc_[lev] - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::Copy(state_new_fc_[lev][idim], avgFaceVel[idim], Physics_Indices::velFirstIndex, 0, - Physics_NumVars::numVelVars_per_dim, nghost_vel); - } - - // fill ghost faces - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - fillBoundaryConditions(state_new_fc_[lev][idim], state_new_fc_[lev][idim], lev, time + 0.5 * dt_lev, quokka::centering::fc, - quokka::direction{idim}, AMRSimulation::InterpHookNone, AMRSimulation::InterpHookNone, - FillPatchType::fillpatch_function); - } - - // copy state_new_fc_[lev] to avgFaceVel - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::Copy(avgFaceVel[idim], state_new_fc_[lev][idim], 0, Physics_Indices::velFirstIndex, - Physics_NumVars::numVelVars_per_dim, nghost_vel); - } - - // advect particles - TracerPC->AdvectWithUmac(avgFaceVel.data(), lev, dt_lev); - } + if (do_tracers != 0) { + // copy avgFaceVel to state_new_fc_[lev] + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::Copy(state_new_fc_[lev][idim], avgFaceVel[idim], Physics_Indices::velFirstIndex, 0, + Physics_NumVars::numVelVars_per_dim, nghost_vel); + } + + // fill ghost faces + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + fillBoundaryConditions(state_new_fc_[lev][idim], state_new_fc_[lev][idim], lev, time + 0.5 * dt_lev, quokka::centering::fc, + quokka::direction{idim}, AMRSimulation::InterpHookNone, AMRSimulation::InterpHookNone, + FillPatchType::fillpatch_function); + } + + // copy state_new_fc_[lev] to avgFaceVel + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::Copy(avgFaceVel[idim], state_new_fc_[lev][idim], 0, Physics_Indices::velFirstIndex, + Physics_NumVars::numVelVars_per_dim, nghost_vel); + } + + // advect particles + TracerPC->AdvectWithUmac(avgFaceVel.data(), lev, dt_lev); + } #endif - // do Strang split source terms (second half-step) - addStrangSplitSourcesWithBuiltin(state_new_cc_[lev], lev, time + dt_lev, 0.5 * dt_lev); + // do Strang split source terms (second half-step) + addStrangSplitSourcesWithBuiltin(state_new_cc_[lev], lev, time + dt_lev, 0.5 * dt_lev); - // check if we have violated the CFL timestep - return !isCflViolated(lev, time, dt_lev); + // check if we have violated the CFL timestep + return !isCflViolated(lev, time, dt_lev); } template void RadhydroSimulation::replaceFluxes(std::array &fluxes, std::array &FOfluxes, - amrex::iMultiFab &redoFlag) + amrex::iMultiFab &redoFlag) { - BL_PROFILE("RadhydroSimulation::replaceFluxes()"); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // loop over dimension - // ensure that flux arrays have the same number of components - AMREX_ASSERT(fluxes[idim].nComp() == FOfluxes[idim].nComp()); - int ncomp = fluxes[idim].nComp(); - - auto const &FOflux_arrs = FOfluxes[idim].const_arrays(); - auto const &redoFlag_arrs = redoFlag.const_arrays(); - auto flux_arrs = fluxes[idim].arrays(); - - // By convention, the fluxes are defined on the left edge of each zone, - // i.e. flux_(i) is the flux *into* zone i through the interface on the - // left of zone i, and -1.0*flux(i+1) is the flux *into* zone i through - // the interface on the right of zone i. - - amrex::IntVect ng{AMREX_D_DECL(1, 1, 1)}; // must include 1 ghost zone - - amrex::ParallelFor(redoFlag, ng, ncomp, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k, int n) noexcept { - if (redoFlag_arrs[bx](i, j, k) == quokka::redoFlag::redo) { - // replace fluxes with first-order ones at the faces of cell (i,j,k) - if (flux_arrs[bx].contains(i, j, k)) { - flux_arrs[bx](i, j, k, n) = FOflux_arrs[bx](i, j, k, n); - } - if (idim == 0) { // x-dir fluxes - if (flux_arrs[bx].contains(i + 1, j, k)) { - flux_arrs[bx](i + 1, j, k, n) = FOflux_arrs[bx](i + 1, j, k, n); - } - } else if (idim == 1) { // y-dir fluxes - if (flux_arrs[bx].contains(i, j + 1, k)) { - flux_arrs[bx](i, j + 1, k, n) = FOflux_arrs[bx](i, j + 1, k, n); - } - } else if (idim == 2) { // z-dir fluxes - if (flux_arrs[bx].contains(i, j, k + 1)) { - flux_arrs[bx](i, j, k + 1, n) = FOflux_arrs[bx](i, j, k + 1, n); - } - } - } - }); - } + BL_PROFILE("RadhydroSimulation::replaceFluxes()"); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // loop over dimension + // ensure that flux arrays have the same number of components + AMREX_ASSERT(fluxes[idim].nComp() == FOfluxes[idim].nComp()); + int ncomp = fluxes[idim].nComp(); + + auto const &FOflux_arrs = FOfluxes[idim].const_arrays(); + auto const &redoFlag_arrs = redoFlag.const_arrays(); + auto flux_arrs = fluxes[idim].arrays(); + + // By convention, the fluxes are defined on the left edge of each zone, + // i.e. flux_(i) is the flux *into* zone i through the interface on the + // left of zone i, and -1.0*flux(i+1) is the flux *into* zone i through + // the interface on the right of zone i. + + amrex::IntVect ng{AMREX_D_DECL(1, 1, 1)}; // must include 1 ghost zone + + amrex::ParallelFor(redoFlag, ng, ncomp, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k, int n) noexcept { + if (redoFlag_arrs[bx](i, j, k) == quokka::redoFlag::redo) { + // replace fluxes with first-order ones at the faces of cell (i,j,k) + if (flux_arrs[bx].contains(i, j, k)) { + flux_arrs[bx](i, j, k, n) = FOflux_arrs[bx](i, j, k, n); + } + if (idim == 0) { // x-dir fluxes + if (flux_arrs[bx].contains(i + 1, j, k)) { + flux_arrs[bx](i + 1, j, k, n) = FOflux_arrs[bx](i + 1, j, k, n); + } + } else if (idim == 1) { // y-dir fluxes + if (flux_arrs[bx].contains(i, j + 1, k)) { + flux_arrs[bx](i, j + 1, k, n) = FOflux_arrs[bx](i, j + 1, k, n); + } + } else if (idim == 2) { // z-dir fluxes + if (flux_arrs[bx].contains(i, j, k + 1)) { + flux_arrs[bx](i, j, k + 1, n) = FOflux_arrs[bx](i, j, k + 1, n); + } + } + } + }); + } } template void RadhydroSimulation::addFluxArrays(std::array &dstfluxes, - std::array &srcfluxes, const int srccomp, const int dstcomp) + std::array &srcfluxes, const int srccomp, const int dstcomp) { - BL_PROFILE("RadhydroSimulation::addFluxArrays()"); + BL_PROFILE("RadhydroSimulation::addFluxArrays()"); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto const &srcflux = srcfluxes[idim]; - auto &dstflux = dstfluxes[idim]; - amrex::Add(dstflux, srcflux, srccomp, dstcomp, srcflux.nComp(), 0); - } + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto const &srcflux = srcfluxes[idim]; + auto &dstflux = dstfluxes[idim]; + amrex::Add(dstflux, srcflux, srccomp, dstcomp, srcflux.nComp(), 0); + } } template auto RadhydroSimulation::expandFluxArrays(std::array &fluxes, const int nstartNew, const int ncompNew) -> std::array { - BL_PROFILE("RadhydroSimulation::expandFluxArrays()"); - - // This is needed because reflux arrays must have the same number of components as - // state_new_cc_[lev] - auto copyFlux = [nstartNew, ncompNew](amrex::FArrayBox const &oldFlux) { - amrex::Box const &fluxRange = oldFlux.box(); - amrex::FArrayBox newFlux(fluxRange, ncompNew, amrex::The_Async_Arena()); - newFlux.setVal(0.); - // copy oldFlux (starting at 0) to newFlux (starting at nstart) - AMREX_ASSERT(ncompNew >= oldFlux.nComp()); - newFlux.copy(oldFlux, 0, nstartNew, oldFlux.nComp()); - return newFlux; - }; - return {AMREX_D_DECL(copyFlux(fluxes[0]), copyFlux(fluxes[1]), copyFlux(fluxes[2]))}; + BL_PROFILE("RadhydroSimulation::expandFluxArrays()"); + + // This is needed because reflux arrays must have the same number of components as + // state_new_cc_[lev] + auto copyFlux = [nstartNew, ncompNew](amrex::FArrayBox const &oldFlux) { + amrex::Box const &fluxRange = oldFlux.box(); + amrex::FArrayBox newFlux(fluxRange, ncompNew, amrex::The_Async_Arena()); + newFlux.setVal(0.); + // copy oldFlux (starting at 0) to newFlux (starting at nstart) + AMREX_ASSERT(ncompNew >= oldFlux.nComp()); + newFlux.copy(oldFlux, 0, nstartNew, oldFlux.nComp()); + return newFlux; + }; + return {AMREX_D_DECL(copyFlux(fluxes[0]), copyFlux(fluxes[1]), copyFlux(fluxes[2]))}; } template auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) -> std::pair, std::array> { - BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); - - auto ba = grids[lev]; - auto dm = dmap[lev]; - const int flatteningGhost = 2; - const int reconstructGhost = 1; - - // allocate temporary MultiFabs - amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); - std::array flatCoefs; - std::array flux; - std::array facevel; - std::array leftState; - std::array rightState; - - for (int idim = 0; idim < 3; ++idim) { - flatCoefs[idim] = amrex::MultiFab(ba, dm, 1, flatteningGhost); - } - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); - rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); - flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); - facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); - } - - // conserved to primitive variables - HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); - - // compute flattening coefficients - AMREX_D_TERM(HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[0], flatteningGhost); - , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[1], flatteningGhost); - , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) - - // compute flux functions - AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars);) - - // synchronization point to prevent MultiFabs from going out of scope - amrex::Gpu::streamSynchronizeAll(); - - // LOW LEVEL DEBUGGING: output all of the temporary MultiFabs - if (lowLevelDebuggingOutput_ == 1) { - // write primitive cell-centered state - std::string plotfile_name = CustomPlotFileName("debug_reconstruction", istep[lev] + 1); - WriteSingleLevelPlotfile(plotfile_name, primVar, componentNames_cc_, geom[lev], 0.0, istep[lev] + 1); - - // write flattening coefficients - std::string flatx_filename = CustomPlotFileName("debug_flattening_x", istep[lev] + 1); - std::string flaty_filename = CustomPlotFileName("debug_flattening_y", istep[lev] + 1); - std::string flatz_filename = CustomPlotFileName("debug_flattening_z", istep[lev] + 1); - amrex::Vector flatCompNames{"chi"}; - WriteSingleLevelPlotfile(flatx_filename, flatCoefs[0], flatCompNames, geom[lev], 0.0, istep[lev] + 1); - WriteSingleLevelPlotfile(flaty_filename, flatCoefs[1], flatCompNames, geom[lev], 0.0, istep[lev] + 1); - WriteSingleLevelPlotfile(flatz_filename, flatCoefs[2], flatCompNames, geom[lev], 0.0, istep[lev] + 1); - - // write L interface states - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string const fullprefix = - amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateL_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(leftState[idim], fullprefix); - } - // write R interface states - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string const fullprefix = - amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateR_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(rightState[idim], fullprefix); - } - } - - // return flux and face-centered velocities - return std::make_pair(std::move(flux), std::move(facevel)); + BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); + + auto ba = grids[lev]; + auto dm = dmap[lev]; + const int flatteningGhost = 2; + const int reconstructGhost = 1; + + // allocate temporary MultiFabs + amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); + std::array flatCoefs; + std::array flux; + std::array facevel; + std::array leftState; + std::array rightState; + + for (int idim = 0; idim < 3; ++idim) { + flatCoefs[idim] = amrex::MultiFab(ba, dm, 1, flatteningGhost); + } + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); + rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); + flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); + facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); + } + + // conserved to primitive variables + HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); + + // compute flattening coefficients + AMREX_D_TERM(HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[0], flatteningGhost); + , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[1], flatteningGhost); + , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) + + // compute flux functions + AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], + reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], + reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], + reconstructGhost, nvars);) + + // synchronization point to prevent MultiFabs from going out of scope + amrex::Gpu::streamSynchronizeAll(); + + // LOW LEVEL DEBUGGING: output all of the temporary MultiFabs + if (lowLevelDebuggingOutput_ == 1) { + // write primitive cell-centered state + std::string plotfile_name = CustomPlotFileName("debug_reconstruction", istep[lev] + 1); + WriteSingleLevelPlotfile(plotfile_name, primVar, componentNames_cc_, geom[lev], 0.0, istep[lev] + 1); + + // write flattening coefficients + std::string flatx_filename = CustomPlotFileName("debug_flattening_x", istep[lev] + 1); + std::string flaty_filename = CustomPlotFileName("debug_flattening_y", istep[lev] + 1); + std::string flatz_filename = CustomPlotFileName("debug_flattening_z", istep[lev] + 1); + amrex::Vector flatCompNames{"chi"}; + WriteSingleLevelPlotfile(flatx_filename, flatCoefs[0], flatCompNames, geom[lev], 0.0, istep[lev] + 1); + WriteSingleLevelPlotfile(flaty_filename, flatCoefs[1], flatCompNames, geom[lev], 0.0, istep[lev] + 1); + WriteSingleLevelPlotfile(flatz_filename, flatCoefs[2], flatCompNames, geom[lev], 0.0, istep[lev] + 1); + + // write L interface states + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string const fullprefix = + amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateL_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(leftState[idim], fullprefix); + } + // write R interface states + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string const fullprefix = + amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateR_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(rightState[idim], fullprefix); + } + } + + // return flux and face-centered velocities + return std::make_pair(std::move(flux), std::move(facevel)); } template template void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, - amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab const &x1Flat, - amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) + amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab const &x1Flat, + amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) { - if (reconstructionOrder_ == 3) { - HydroSystem::template ReconstructStatesPPM(primVar, leftState, rightState, ng_reconstruct, nvars); - } else if (reconstructionOrder_ == 2) { - HydroSystem::template ReconstructStatesPLM(primVar, leftState, rightState, ng_reconstruct, nvars); - } else if (reconstructionOrder_ == 1) { - HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); - } else { - amrex::Abort("Invalid reconstruction order specified!"); - } - - // cell-centered kernel - HydroSystem::template FlattenShocks(primVar, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); - - // interface-centered kernel - if constexpr (Physics_Traits::is_mhd_enabled) { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); - } else { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); - } + if (reconstructionOrder_ == 3) { + HydroSystem::template ReconstructStatesPPM(primVar, leftState, rightState, ng_reconstruct, nvars); + } else if (reconstructionOrder_ == 2) { + HydroSystem::template ReconstructStatesPLM(primVar, leftState, rightState, ng_reconstruct, nvars); + } else if (reconstructionOrder_ == 1) { + HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); + } else { + amrex::Abort("Invalid reconstruction order specified!"); + } + + // cell-centered kernel + HydroSystem::template FlattenShocks(primVar, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); + + // interface-centered kernel + if constexpr (Physics_Traits::is_mhd_enabled) { + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + } else { + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + } } template auto RadhydroSimulation::computeFOHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) -> std::pair, std::array> { - BL_PROFILE("RadhydroSimulation::computeFOHydroFluxes()"); - - auto ba = grids[lev]; - auto dm = dmap[lev]; - const int reconstructRange = 1; - - // allocate temporary MultiFabs - amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); - std::array flux; - std::array facevel; - std::array leftState; - std::array rightState; - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); - rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); - flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); - facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); - } - - // conserved to primitive variables - HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); - - // compute flux functions - AMREX_D_TERM(hydroFOFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], reconstructRange, nvars); - , hydroFOFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], reconstructRange, nvars); - , hydroFOFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], reconstructRange, nvars);) - - // synchronization point to prevent MultiFabs from going out of scope - amrex::Gpu::streamSynchronizeAll(); - - // return flux and face-centered velocities - return std::make_pair(std::move(flux), std::move(facevel)); + BL_PROFILE("RadhydroSimulation::computeFOHydroFluxes()"); + + auto ba = grids[lev]; + auto dm = dmap[lev]; + const int reconstructRange = 1; + + // allocate temporary MultiFabs + amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); + std::array flux; + std::array facevel; + std::array leftState; + std::array rightState; + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); + rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); + flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); + facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); + } + + // conserved to primitive variables + HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); + + // compute flux functions + AMREX_D_TERM(hydroFOFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], reconstructRange, nvars); + , hydroFOFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], reconstructRange, nvars); + , hydroFOFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], reconstructRange, nvars);) + + // synchronization point to prevent MultiFabs from going out of scope + amrex::Gpu::streamSynchronizeAll(); + + // return flux and face-centered velocities + return std::make_pair(std::move(flux), std::move(facevel)); } template template void RadhydroSimulation::hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, - amrex::MultiFab &flux, amrex::MultiFab &faceVel, const int ng_reconstruct, const int nvars) + amrex::MultiFab &flux, amrex::MultiFab &faceVel, const int ng_reconstruct, const int nvars) { - // donor-cell reconstruction - HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); - // LLF solver - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + // donor-cell reconstruction + HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); + // LLF solver + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); } template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew) { - // copy radiation state variables from stateNew to stateOld - amrex::MultiFab::Copy(stateOld, stateNew, nstartHyperbolic_, nstartHyperbolic_, ncompHyperbolic_, 0); + // copy radiation state variables from stateNew to stateOld + amrex::MultiFab::Copy(stateOld, stateNew, nstartHyperbolic_, nstartHyperbolic_, ncompHyperbolic_, 0); } template void RadhydroSimulation::subcycleRadiationAtLevel(int lev, amrex::Real time, amrex::Real dt_lev_hydro, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine) + amrex::YAFluxRegister *fr_as_fine) { - // compute radiation timestep - int nsubSteps = 0; - amrex::Real dt_radiation = NAN; - - if (Physics_Traits::is_hydro_enabled && !(constantDt_ > 0.)) { - // adjust to get integer number of substeps - nsubSteps = computeNumberOfRadiationSubsteps(lev, dt_lev_hydro); - dt_radiation = dt_lev_hydro / static_cast(nsubSteps); - } else { // no hydro, or using constant dt (this is necessary for radiation test problems) - dt_radiation = dt_lev_hydro; - nsubSteps = 1; - } - - if (Verbose() != 0) { - amrex::Print() << "\tRadiation substeps: " << nsubSteps << "\tdt: " << dt_radiation << "\n"; - } - AMREX_ALWAYS_ASSERT(nsubSteps >= 1); - AMREX_ALWAYS_ASSERT(nsubSteps <= (maxSubsteps_ + 1)); - AMREX_ALWAYS_ASSERT(dt_radiation > 0.0); - - // perform subcycle - auto const &dx = geom[lev].CellSizeArray(); - amrex::Real time_subcycle = time; - for (int i = 0; i < nsubSteps; ++i) { - if (i > 0) { - // since we are starting a new substep, we need to copy radiation state from - // new state vector to old state vector - // (this is not necessary for the i=0 substep because we have already swapped - // the full hydro+radiation state vectors at the beginning of the level advance) - swapRadiationState(state_old_cc_[lev], state_new_cc_[lev]); - } - - // We use the IMEX PD-ARS scheme to evolve the radiation subsystem and radiation-matter coupling. - - // Stage 1: advance hyperbolic radiation subsystem using Forward Euler method, starting from state_old_cc_ to state_new_cc_ - advanceRadiationForwardEuler(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); - - // new radiation state is stored in state_new_cc_ - // new hydro state is stored in state_new_cc_ (always the case during radiation update) - - if constexpr (IMEX_a22 > 0.0) { - // matter-radiation exchange source terms of stage 1 - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto const &prob_lo = geom[lev].ProbLoArray(); - auto const &prob_hi = geom[lev].ProbHiArray(); - // update state_new_cc_[lev] in place (updates both radiation and hydro vars) - // Note that only a fraction (IMEX_a32) of the matter-radiation exchange source terms are added to hydro. This ensures that the - // hydro properties get to t + IMEX_a32 dt in terms of matter-radiation exchange. - operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 1, dx, prob_lo, prob_hi); - } - } - - // Stage 2: advance hyperbolic radiation subsystem using midpoint RK2 method, starting from state_old_cc_ to state_new_cc_ - advanceRadiationMidpointRK2(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); - - // new radiation state is stored in state_new_cc_ - // new hydro state is stored in state_new_cc_ (always the case during radiation update) - - // Add the matter-radiation exchange source terms to the radiation subsystem and evolve by (1 - IMEX_a32) * dt - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto const &prob_lo = geom[lev].ProbLoArray(); - auto const &prob_hi = geom[lev].ProbHiArray(); - // update state_new_cc_[lev] in place (updates both radiation and hydro vars) - operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 2, dx, prob_lo, prob_hi); - } - - // new hydro+radiation state is stored in state_new_cc_ - - // update 'time_subcycle' - time_subcycle += dt_radiation; - - // update cell update counter - radiationCellUpdates_ += CountCells(lev); // keep track of number of cell updates - } + // compute radiation timestep + int nsubSteps = 0; + amrex::Real dt_radiation = NAN; + + if (Physics_Traits::is_hydro_enabled && !(constantDt_ > 0.)) { + // adjust to get integer number of substeps + nsubSteps = computeNumberOfRadiationSubsteps(lev, dt_lev_hydro); + dt_radiation = dt_lev_hydro / static_cast(nsubSteps); + } else { // no hydro, or using constant dt (this is necessary for radiation test problems) + dt_radiation = dt_lev_hydro; + nsubSteps = 1; + } + + if (Verbose() != 0) { + amrex::Print() << "\tRadiation substeps: " << nsubSteps << "\tdt: " << dt_radiation << "\n"; + } + AMREX_ALWAYS_ASSERT(nsubSteps >= 1); + AMREX_ALWAYS_ASSERT(nsubSteps <= (maxSubsteps_ + 1)); + AMREX_ALWAYS_ASSERT(dt_radiation > 0.0); + + // perform subcycle + auto const &dx = geom[lev].CellSizeArray(); + amrex::Real time_subcycle = time; + for (int i = 0; i < nsubSteps; ++i) { + if (i > 0) { + // since we are starting a new substep, we need to copy radiation state from + // new state vector to old state vector + // (this is not necessary for the i=0 substep because we have already swapped + // the full hydro+radiation state vectors at the beginning of the level advance) + swapRadiationState(state_old_cc_[lev], state_new_cc_[lev]); + } + + // We use the IMEX PD-ARS scheme to evolve the radiation subsystem and radiation-matter coupling. + + // Stage 1: advance hyperbolic radiation subsystem using Forward Euler method, starting from state_old_cc_ to state_new_cc_ + advanceRadiationForwardEuler(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); + + // new radiation state is stored in state_new_cc_ + // new hydro state is stored in state_new_cc_ (always the case during radiation update) + + if constexpr (IMEX_a22 > 0.0) { + // matter-radiation exchange source terms of stage 1 + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &prob_lo = geom[lev].ProbLoArray(); + auto const &prob_hi = geom[lev].ProbHiArray(); + // update state_new_cc_[lev] in place (updates both radiation and hydro vars) + // Note that only a fraction (IMEX_a32) of the matter-radiation exchange source terms are added to hydro. This ensures that the + // hydro properties get to t + IMEX_a32 dt in terms of matter-radiation exchange. + operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 1, dx, prob_lo, prob_hi); + } + } + + // Stage 2: advance hyperbolic radiation subsystem using midpoint RK2 method, starting from state_old_cc_ to state_new_cc_ + advanceRadiationMidpointRK2(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); + + // new radiation state is stored in state_new_cc_ + // new hydro state is stored in state_new_cc_ (always the case during radiation update) + + // Add the matter-radiation exchange source terms to the radiation subsystem and evolve by (1 - IMEX_a32) * dt + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &prob_lo = geom[lev].ProbLoArray(); + auto const &prob_hi = geom[lev].ProbHiArray(); + // update state_new_cc_[lev] in place (updates both radiation and hydro vars) + operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 2, dx, prob_lo, prob_hi); + } + + // new hydro+radiation state is stored in state_new_cc_ + + // update 'time_subcycle' + time_subcycle += dt_radiation; + + // update cell update counter + radiationCellUpdates_ += CountCells(lev); // keep track of number of cell updates + } } template void RadhydroSimulation::advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int const iter_count, - int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine) + int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine) { - if (Verbose()) { - amrex::Print() << "\tsubstep " << iter_count << " t = " << time << std::endl; - } - - // get cell sizes - auto const &dx = geom[lev].CellSizeArray(); - - // We use the RK2-SSP method here. It needs two registers: one to store the old timestep, - // and another to store the intermediate stage (which is reused for the final stage). - - // update ghost zones [old timestep] - fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 1 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); - - // Stage 1 of RK2-SSP - RadSystem::PredictStep( - stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } - - // update ghost zones [intermediate stage stored in state_new_cc_] - fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 2 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateInter = state_new_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); - - // Stage 2 of RK2-SSP - RadSystem::AddFluxesRK2( - stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } + if (Verbose()) { + amrex::Print() << "\tsubstep " << iter_count << " t = " << time << std::endl; + } + + // get cell sizes + auto const &dx = geom[lev].CellSizeArray(); + + // We use the RK2-SSP method here. It needs two registers: one to store the old timestep, + // and another to store the intermediate stage (which is reused for the final stage). + + // update ghost zones [old timestep] + fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 1 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + + // Stage 1 of RK2-SSP + RadSystem::PredictStep( + stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } + + // update ghost zones [intermediate stage stored in state_new_cc_] + fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 2 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateInter = state_new_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); + + // Stage 2 of RK2-SSP + RadSystem::AddFluxesRK2( + stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } } template void RadhydroSimulation::advanceRadiationForwardEuler(int lev, amrex::Real time, amrex::Real dt_radiation, int const /*iter_count*/, - int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) + int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) { - // get cell sizes - auto const &dx = geom[lev].CellSizeArray(); - - // update ghost zones [old timestep] - fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 1 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); - - // Stage 1 of RK2-SSP - RadSystem::PredictStep( - stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } + // get cell sizes + auto const &dx = geom[lev].CellSizeArray(); + + // update ghost zones [old timestep] + fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 1 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + + // Stage 1 of RK2-SSP + RadSystem::PredictStep( + stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } } template void RadhydroSimulation::advanceRadiationMidpointRK2(int lev, amrex::Real time, amrex::Real dt_radiation, int const /*iter_count*/, - int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) + int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) { - auto const &dx = geom[lev].CellSizeArray(); - - // update ghost zones [intermediate stage stored in state_new_cc_] - fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 2 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateInter = state_new_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); - - // Stage 2 of RK2-SSP - RadSystem::AddFluxesRK2( - stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, - {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArraysOld[0].const_array(), fluxDiffusiveArraysOld[1].const_array(), fluxDiffusiveArraysOld[2].const_array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } + auto const &dx = geom[lev].CellSizeArray(); + + // update ghost zones [intermediate stage stored in state_new_cc_] + fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 2 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateInter = state_new_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); + + // Stage 2 of RK2-SSP + RadSystem::AddFluxesRK2( + stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, + {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArraysOld[0].const_array(), fluxDiffusiveArraysOld[1].const_array(), fluxDiffusiveArraysOld[2].const_array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } } template void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, const amrex::Real time, - const double dt, const int stage, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo, - amrex::GpuArray const &prob_hi) + const double dt, const int stage, amrex::GpuArray const &dx, + amrex::GpuArray const &prob_lo, + amrex::GpuArray const &prob_hi) { - amrex::FArrayBox radEnergySource(indexRange, Physics_Traits::nGroups, - amrex::The_Async_Arena()); // cell-centered scalar + amrex::FArrayBox radEnergySource(indexRange, Physics_Traits::nGroups, + amrex::The_Async_Arena()); // cell-centered scalar - radEnergySource.setVal(0.); + radEnergySource.setVal(0.); - // cell-centered radiation energy source - RadSystem::SetRadEnergySource(radEnergySource.array(), indexRange, dx, prob_lo, prob_hi, time + dt); + // cell-centered radiation energy source + RadSystem::SetRadEnergySource(radEnergySource.array(), indexRange, dx, prob_lo, prob_hi, time + dt); - // cell-centered source terms - RadSystem::AddSourceTerms(stateNew, radEnergySource.const_array(), indexRange, dt, stage); + // cell-centered source terms + RadSystem::AddSourceTerms(stateNew, radEnergySource.const_array(), indexRange, dt, stage); } template auto RadhydroSimulation::computeRadiationFluxes(amrex::Array4 const &consVar, const amrex::Box &indexRange, const int nvars, - amrex::GpuArray dx) + amrex::GpuArray dx) -> std::tuple, std::array> { - amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, 0); - amrex::FArrayBox x1Flux(x1FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in x - amrex::FArrayBox x1FluxDiffusive(x1FluxRange, nvars, amrex::The_Async_Arena()); + amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, 0); + amrex::FArrayBox x1Flux(x1FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in x + amrex::FArrayBox x1FluxDiffusive(x1FluxRange, nvars, amrex::The_Async_Arena()); #if (AMREX_SPACEDIM >= 2) - amrex::Box const &x2FluxRange = amrex::surroundingNodes(indexRange, 1); - amrex::FArrayBox x2Flux(x2FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in y - amrex::FArrayBox x2FluxDiffusive(x2FluxRange, nvars, amrex::The_Async_Arena()); + amrex::Box const &x2FluxRange = amrex::surroundingNodes(indexRange, 1); + amrex::FArrayBox x2Flux(x2FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in y + amrex::FArrayBox x2FluxDiffusive(x2FluxRange, nvars, amrex::The_Async_Arena()); #endif #if (AMREX_SPACEDIM == 3) - amrex::Box const &x3FluxRange = amrex::surroundingNodes(indexRange, 2); - amrex::FArrayBox x3Flux(x3FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in z - amrex::FArrayBox x3FluxDiffusive(x3FluxRange, nvars, amrex::The_Async_Arena()); + amrex::Box const &x3FluxRange = amrex::surroundingNodes(indexRange, 2); + amrex::FArrayBox x3Flux(x3FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in z + amrex::FArrayBox x3FluxDiffusive(x3FluxRange, nvars, amrex::The_Async_Arena()); #endif - AMREX_D_TERM(fluxFunction(consVar, x1Flux, x1FluxDiffusive, indexRange, nvars, dx); - , fluxFunction(consVar, x2Flux, x2FluxDiffusive, indexRange, nvars, dx); - , fluxFunction(consVar, x3Flux, x3FluxDiffusive, indexRange, nvars, dx);) + AMREX_D_TERM(fluxFunction(consVar, x1Flux, x1FluxDiffusive, indexRange, nvars, dx); + , fluxFunction(consVar, x2Flux, x2FluxDiffusive, indexRange, nvars, dx); + , fluxFunction(consVar, x3Flux, x3FluxDiffusive, indexRange, nvars, dx);) - std::array fluxArrays = {AMREX_D_DECL(std::move(x1Flux), std::move(x2Flux), std::move(x3Flux))}; - std::array fluxDiffusiveArrays{ - AMREX_D_DECL(std::move(x1FluxDiffusive), std::move(x2FluxDiffusive), std::move(x3FluxDiffusive))}; + std::array fluxArrays = {AMREX_D_DECL(std::move(x1Flux), std::move(x2Flux), std::move(x3Flux))}; + std::array fluxDiffusiveArrays{ + AMREX_D_DECL(std::move(x1FluxDiffusive), std::move(x2FluxDiffusive), std::move(x3FluxDiffusive))}; - return std::make_tuple(std::move(fluxArrays), std::move(fluxDiffusiveArrays)); + return std::make_tuple(std::move(fluxArrays), std::move(fluxDiffusiveArrays)); } template template void RadhydroSimulation::fluxFunction(amrex::Array4 const &consState, amrex::FArrayBox &x1Flux, amrex::FArrayBox &x1FluxDiffusive, - const amrex::Box &indexRange, const int nvars, amrex::GpuArray dx) + const amrex::Box &indexRange, const int nvars, amrex::GpuArray dx) { - int dir = 0; - if constexpr (DIR == FluxDir::X1) { - dir = 0; - } else if constexpr (DIR == FluxDir::X2) { - dir = 1; - } else if constexpr (DIR == FluxDir::X3) { - dir = 2; - } - - // extend box to include ghost zones - amrex::Box const &ghostRange = amrex::grow(indexRange, nghost_cc_); - // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to - // work. - amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); - amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, dir); - - amrex::FArrayBox primVar(ghostRange, nvars, amrex::The_Async_Arena()); - amrex::FArrayBox x1LeftState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); - amrex::FArrayBox x1RightState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); - - // cell-centered kernel - RadSystem::ConservedToPrimitive(consState, primVar.array(), ghostRange); - - if (radiationReconstructionOrder_ == 3) { - // mixed interface/cell-centered kernel - RadSystem::template ReconstructStatesPPM(primVar.array(), x1LeftState.array(), x1RightState.array(), reconstructRange, - x1ReconstructRange, nvars); - } else if (radiationReconstructionOrder_ == 2) { - // PLM and donor cell are interface-centered kernels - RadSystem::template ReconstructStatesPLM(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, nvars); - } else if (radiationReconstructionOrder_ == 1) { - RadSystem::template ReconstructStatesConstant(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, - nvars); - } else { - amrex::Abort("Invalid reconstruction order for radiation variables! Aborting..."); - } - - // interface-centered kernel - amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, dir); - RadSystem::template ComputeFluxes(x1Flux.array(), x1FluxDiffusive.array(), x1LeftState.array(), x1RightState.array(), x1FluxRange, - consState, - dx); // watch out for argument order!! + int dir = 0; + if constexpr (DIR == FluxDir::X1) { + dir = 0; + } else if constexpr (DIR == FluxDir::X2) { + dir = 1; + } else if constexpr (DIR == FluxDir::X3) { + dir = 2; + } + + // extend box to include ghost zones + amrex::Box const &ghostRange = amrex::grow(indexRange, nghost_cc_); + // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to + // work. + amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); + amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, dir); + + amrex::FArrayBox primVar(ghostRange, nvars, amrex::The_Async_Arena()); + amrex::FArrayBox x1LeftState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); + amrex::FArrayBox x1RightState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); + + // cell-centered kernel + RadSystem::ConservedToPrimitive(consState, primVar.array(), ghostRange); + + if (radiationReconstructionOrder_ == 3) { + // mixed interface/cell-centered kernel + RadSystem::template ReconstructStatesPPM(primVar.array(), x1LeftState.array(), x1RightState.array(), reconstructRange, + x1ReconstructRange, nvars); + } else if (radiationReconstructionOrder_ == 2) { + // PLM and donor cell are interface-centered kernels + RadSystem::template ReconstructStatesPLM(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, nvars); + } else if (radiationReconstructionOrder_ == 1) { + RadSystem::template ReconstructStatesConstant(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, + nvars); + } else { + amrex::Abort("Invalid reconstruction order for radiation variables! Aborting..."); + } + + // interface-centered kernel + amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, dir); + RadSystem::template ComputeFluxes(x1Flux.array(), x1FluxDiffusive.array(), x1LeftState.array(), x1RightState.array(), x1FluxRange, + consState, + dx); // watch out for argument order!! } #endif // RADIATION_SIMULATION_HPP_ From e49e3780b633e131103fad075c495f1789172073 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:26:11 +0000 Subject: [PATCH 127/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/RadhydroSimulation.hpp | 3000 ++++++++++++++++++------------------ 1 file changed, 1500 insertions(+), 1500 deletions(-) diff --git a/src/RadhydroSimulation.hpp b/src/RadhydroSimulation.hpp index 696978e92..2793efd38 100644 --- a/src/RadhydroSimulation.hpp +++ b/src/RadhydroSimulation.hpp @@ -75,371 +75,371 @@ template class RadhydroSimulation : public AMRSimulation { public: - using AMRSimulation::state_old_cc_; - using AMRSimulation::state_new_cc_; - using AMRSimulation::max_signal_speed_; - using AMRSimulation::state_old_fc_; - using AMRSimulation::state_new_fc_; - using AMRSimulation::TracerPC; - - using AMRSimulation::nghost_cc_; - using AMRSimulation::areInitialConditionsDefined_; - using AMRSimulation::BCs_cc_; - using AMRSimulation::BCs_fc_; - using AMRSimulation::componentNames_cc_; - using AMRSimulation::componentNames_fc_; - using AMRSimulation::cflNumber_; - using AMRSimulation::fillBoundaryConditions; - using AMRSimulation::CustomPlotFileName; - using AMRSimulation::geom; - using AMRSimulation::grids; - using AMRSimulation::dmap; - using AMRSimulation::istep; - using AMRSimulation::flux_reg_; - using AMRSimulation::incrementFluxRegisters; - using AMRSimulation::finest_level; - using AMRSimulation::finestLevel; - using AMRSimulation::do_reflux; - using AMRSimulation::do_tracers; - using AMRSimulation::Verbose; - using AMRSimulation::constantDt_; - using AMRSimulation::boxArray; - using AMRSimulation::DistributionMap; - using AMRSimulation::refRatio; - using AMRSimulation::cellUpdates_; - using AMRSimulation::CountCells; - using AMRSimulation::WriteCheckpointFile; - using AMRSimulation::GetData; - using AMRSimulation::FillPatchWithData; - using AMRSimulation::Gconst_; - - using AMRSimulation::densityFloor_; - using AMRSimulation::tempFloor_; - using AMRSimulation::tempCeiling_; - using AMRSimulation::speedCeiling_; - - SimulationData userData_; - - int enableCooling_ = 0; - int enableChemistry_ = 0; - Real max_density_allowed = std::numeric_limits::max(); - Real min_density_allowed = std::numeric_limits::min(); - - quokka::cooling::cloudy_tables cloudyTables_; - std::string coolingTableFilename_{}; - - static constexpr int nvarTotal_cc_ = Physics_Indices::nvarTotal_cc; - static constexpr int ncompHydro_ = HydroSystem::nvar_; // hydro - static constexpr int ncompHyperbolic_ = RadSystem::nvarHyperbolic_; - static constexpr int nstartHyperbolic_ = RadSystem::nstartHyperbolic_; - - amrex::Real radiationCflNumber_ = 0.3; - int maxSubsteps_ = 10; // maximum number of radiation subcycles per hydro step - - bool computeReferenceSolution_ = false; - amrex::Real errorNorm_ = NAN; - amrex::Real pressureFloor_ = 0.; - - int lowLevelDebuggingOutput_ = 0; // 0 == do nothing; 1 == output intermediate multifabs used in hydro each timestep (ONLY USE FOR DEBUGGING) - int integratorOrder_ = 2; // 1 == forward Euler; 2 == RK2-SSP (default) - int reconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) - int radiationReconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) - int useDualEnergy_ = 1; // 0 == disabled; 1 == use auxiliary internal energy equation (default) - int abortOnFofcFailure_ = 1; // 0 == keep going, 1 == abort hydro advance if FOFC fails - amrex::Real artificialViscosityK_ = 0.; // artificial viscosity coefficient (default == None) - - amrex::Long radiationCellUpdates_ = 0; // total number of radiation cell-updates - - // member functions - explicit RadhydroSimulation(amrex::Vector &BCs_cc, amrex::Vector &BCs_fc) : AMRSimulation(BCs_cc, BCs_fc) - { - initialize(); - } - - explicit RadhydroSimulation(amrex::Vector &BCs_cc) : AMRSimulation(BCs_cc) { initialize(); } - - inline void initialize() - { - defineComponentNames(); - // read in runtime parameters - readParmParse(); - // set gamma - amrex::ParmParse eos("eos"); - eos.add("eos_gamma", quokka::EOS_Traits::gamma); - // initialize Microphysics params - init_extern_parameters(); - // initialize Microphysics EOS - amrex::Real small_temp = 1e-10; - amrex::Real small_dens = 1e-100; - eos_init(small_temp, small_dens); - } - - [[nodiscard]] static auto getScalarVariableNames() -> std::vector; - void defineComponentNames(); - void readParmParse(); - - void checkHydroStates(amrex::MultiFab &mf, char const *file, int line); - void computeMaxSignalLocal(int level) override; - auto computeExtraPhysicsTimestep(int lev) -> amrex::Real override; - void preCalculateInitialConditions() override; - void setInitialConditionsOnGrid(quokka::grid grid_elem) override; - void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) override; - void createInitialParticles() override; - void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) override; - void computeAfterTimestep() override; - void computeAfterLevelAdvance(int lev, amrex::Real time, amrex::Real dt_lev, int /*ncycle*/); - void computeAfterEvolve(amrex::Vector &initSumCons) override; - void computeReferenceSolution(amrex::MultiFab &ref, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo); - - // compute derived variables - void ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, int ncomp) const override; - - // compute projected vars - [[nodiscard]] auto ComputeProjections(int dir) const -> std::unordered_map> override; - - // compute statistics - auto ComputeStatistics() -> std::map override; - - // fix-up states - void FixupState(int level) override; - - // implement FillPatch function - void FillPatch(int lev, amrex::Real time, amrex::MultiFab &mf, int icomp, int ncomp, quokka::centering cen, quokka::direction dir, - FillPatchType fptype) override; - - // functions to operate on state vector before/after interpolating between levels - static void PreInterpState(amrex::MultiFab &mf, int scomp, int ncomp); - static void PostInterpState(amrex::MultiFab &mf, int scomp, int ncomp); - - // compute axis-aligned 1D profile of user_f(x, y, z) - template auto computeAxisAlignedProfile(int axis, F const &user_f) -> amrex::Gpu::HostVector; - - // tag cells for refinement - void ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override; - - // fill rhs for Poisson solve - void fillPoissonRhsAtLevel(amrex::MultiFab &rhs, int lev) override; - - // add gravitational acceleration to hydro state - void applyPoissonGravityAtLevel(amrex::MultiFab const &phi, int lev, amrex::Real dt) override; - - void addFluxArrays(std::array &dstfluxes, std::array &srcfluxes, const int srccomp, - const int dstcomp); - - auto expandFluxArrays(std::array &fluxes, int nstartNew, int ncompNew) - -> std::array; - - void printCoordinates(int lev, const amrex::IntVect &cell_idx); - - void advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - - auto advanceHydroAtLevel(amrex::MultiFab &state_old_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, - amrex::Real time, amrex::Real dt_lev) -> bool; - - void addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); - void addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); - - auto isCflViolated(int lev, amrex::Real time, amrex::Real dt_actual) -> bool; - - // radiation subcycle - void swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew); - auto computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int; - void advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, - amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); - void advanceRadiationForwardEuler(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - void advanceRadiationMidpointRK2(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - - void subcycleRadiationAtLevel(int lev, amrex::Real time, amrex::Real dt_lev_hydro, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine); - - void operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, amrex::Real time, double dt, int stage, - amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, - amrex::GpuArray const &prob_hi); - - auto computeRadiationFluxes(amrex::Array4 const &consVar, const amrex::Box &indexRange, int nvars, - amrex::GpuArray dx) - -> std::tuple, std::array>; - - auto computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) - -> std::pair, std::array>; + using AMRSimulation::state_old_cc_; + using AMRSimulation::state_new_cc_; + using AMRSimulation::max_signal_speed_; + using AMRSimulation::state_old_fc_; + using AMRSimulation::state_new_fc_; + using AMRSimulation::TracerPC; + + using AMRSimulation::nghost_cc_; + using AMRSimulation::areInitialConditionsDefined_; + using AMRSimulation::BCs_cc_; + using AMRSimulation::BCs_fc_; + using AMRSimulation::componentNames_cc_; + using AMRSimulation::componentNames_fc_; + using AMRSimulation::cflNumber_; + using AMRSimulation::fillBoundaryConditions; + using AMRSimulation::CustomPlotFileName; + using AMRSimulation::geom; + using AMRSimulation::grids; + using AMRSimulation::dmap; + using AMRSimulation::istep; + using AMRSimulation::flux_reg_; + using AMRSimulation::incrementFluxRegisters; + using AMRSimulation::finest_level; + using AMRSimulation::finestLevel; + using AMRSimulation::do_reflux; + using AMRSimulation::do_tracers; + using AMRSimulation::Verbose; + using AMRSimulation::constantDt_; + using AMRSimulation::boxArray; + using AMRSimulation::DistributionMap; + using AMRSimulation::refRatio; + using AMRSimulation::cellUpdates_; + using AMRSimulation::CountCells; + using AMRSimulation::WriteCheckpointFile; + using AMRSimulation::GetData; + using AMRSimulation::FillPatchWithData; + using AMRSimulation::Gconst_; + + using AMRSimulation::densityFloor_; + using AMRSimulation::tempFloor_; + using AMRSimulation::tempCeiling_; + using AMRSimulation::speedCeiling_; + + SimulationData userData_; + + int enableCooling_ = 0; + int enableChemistry_ = 0; + Real max_density_allowed = std::numeric_limits::max(); + Real min_density_allowed = std::numeric_limits::min(); + + quokka::cooling::cloudy_tables cloudyTables_; + std::string coolingTableFilename_{}; + + static constexpr int nvarTotal_cc_ = Physics_Indices::nvarTotal_cc; + static constexpr int ncompHydro_ = HydroSystem::nvar_; // hydro + static constexpr int ncompHyperbolic_ = RadSystem::nvarHyperbolic_; + static constexpr int nstartHyperbolic_ = RadSystem::nstartHyperbolic_; + + amrex::Real radiationCflNumber_ = 0.3; + int maxSubsteps_ = 10; // maximum number of radiation subcycles per hydro step + + bool computeReferenceSolution_ = false; + amrex::Real errorNorm_ = NAN; + amrex::Real pressureFloor_ = 0.; + + int lowLevelDebuggingOutput_ = 0; // 0 == do nothing; 1 == output intermediate multifabs used in hydro each timestep (ONLY USE FOR DEBUGGING) + int integratorOrder_ = 2; // 1 == forward Euler; 2 == RK2-SSP (default) + int reconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) + int radiationReconstructionOrder_ = 3; // 1 == donor cell; 2 == PLM; 3 == PPM (default) + int useDualEnergy_ = 1; // 0 == disabled; 1 == use auxiliary internal energy equation (default) + int abortOnFofcFailure_ = 1; // 0 == keep going, 1 == abort hydro advance if FOFC fails + amrex::Real artificialViscosityK_ = 0.; // artificial viscosity coefficient (default == None) + + amrex::Long radiationCellUpdates_ = 0; // total number of radiation cell-updates + + // member functions + explicit RadhydroSimulation(amrex::Vector &BCs_cc, amrex::Vector &BCs_fc) : AMRSimulation(BCs_cc, BCs_fc) + { + initialize(); + } + + explicit RadhydroSimulation(amrex::Vector &BCs_cc) : AMRSimulation(BCs_cc) { initialize(); } + + inline void initialize() + { + defineComponentNames(); + // read in runtime parameters + readParmParse(); + // set gamma + amrex::ParmParse eos("eos"); + eos.add("eos_gamma", quokka::EOS_Traits::gamma); + // initialize Microphysics params + init_extern_parameters(); + // initialize Microphysics EOS + amrex::Real small_temp = 1e-10; + amrex::Real small_dens = 1e-100; + eos_init(small_temp, small_dens); + } + + [[nodiscard]] static auto getScalarVariableNames() -> std::vector; + void defineComponentNames(); + void readParmParse(); + + void checkHydroStates(amrex::MultiFab &mf, char const *file, int line); + void computeMaxSignalLocal(int level) override; + auto computeExtraPhysicsTimestep(int lev) -> amrex::Real override; + void preCalculateInitialConditions() override; + void setInitialConditionsOnGrid(quokka::grid grid_elem) override; + void setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) override; + void createInitialParticles() override; + void advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) override; + void computeAfterTimestep() override; + void computeAfterLevelAdvance(int lev, amrex::Real time, amrex::Real dt_lev, int /*ncycle*/); + void computeAfterEvolve(amrex::Vector &initSumCons) override; + void computeReferenceSolution(amrex::MultiFab &ref, amrex::GpuArray const &dx, + amrex::GpuArray const &prob_lo); + + // compute derived variables + void ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, int ncomp) const override; + + // compute projected vars + [[nodiscard]] auto ComputeProjections(int dir) const -> std::unordered_map> override; + + // compute statistics + auto ComputeStatistics() -> std::map override; + + // fix-up states + void FixupState(int level) override; + + // implement FillPatch function + void FillPatch(int lev, amrex::Real time, amrex::MultiFab &mf, int icomp, int ncomp, quokka::centering cen, quokka::direction dir, + FillPatchType fptype) override; + + // functions to operate on state vector before/after interpolating between levels + static void PreInterpState(amrex::MultiFab &mf, int scomp, int ncomp); + static void PostInterpState(amrex::MultiFab &mf, int scomp, int ncomp); + + // compute axis-aligned 1D profile of user_f(x, y, z) + template auto computeAxisAlignedProfile(int axis, F const &user_f) -> amrex::Gpu::HostVector; + + // tag cells for refinement + void ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override; + + // fill rhs for Poisson solve + void fillPoissonRhsAtLevel(amrex::MultiFab &rhs, int lev) override; + + // add gravitational acceleration to hydro state + void applyPoissonGravityAtLevel(amrex::MultiFab const &phi, int lev, amrex::Real dt) override; + + void addFluxArrays(std::array &dstfluxes, std::array &srcfluxes, const int srccomp, + const int dstcomp); + + auto expandFluxArrays(std::array &fluxes, int nstartNew, int ncompNew) + -> std::array; + + void printCoordinates(int lev, const amrex::IntVect &cell_idx); + + void advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + + auto advanceHydroAtLevel(amrex::MultiFab &state_old_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, int lev, + amrex::Real time, amrex::Real dt_lev) -> bool; + + void addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); + void addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt_lev); + + auto isCflViolated(int lev, amrex::Real time, amrex::Real dt_actual) -> bool; + + // radiation subcycle + void swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew); + auto computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int; + void advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, + amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine); + void advanceRadiationForwardEuler(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + void advanceRadiationMidpointRK2(int lev, amrex::Real time, amrex::Real dt_radiation, int iter_count, int nsubsteps, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + + void subcycleRadiationAtLevel(int lev, amrex::Real time, amrex::Real dt_lev_hydro, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine); + + void operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, amrex::Real time, double dt, int stage, + amrex::GpuArray const &dx, amrex::GpuArray const &prob_lo, + amrex::GpuArray const &prob_hi); + + auto computeRadiationFluxes(amrex::Array4 const &consVar, const amrex::Box &indexRange, int nvars, + amrex::GpuArray dx) + -> std::tuple, std::array>; + + auto computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) + -> std::pair, std::array>; - auto computeFOHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) - -> std::pair, std::array>; + auto computeFOHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) + -> std::pair, std::array>; - template - void fluxFunction(amrex::Array4 const &consState, amrex::FArrayBox &x1Flux, amrex::FArrayBox &x1FluxDiffusive, - const amrex::Box &indexRange, int nvars, amrex::GpuArray dx); + template + void fluxFunction(amrex::Array4 const &consState, amrex::FArrayBox &x1Flux, amrex::FArrayBox &x1FluxDiffusive, + const amrex::Box &indexRange, int nvars, amrex::GpuArray dx); - template - void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, - int ng_reconstruct, int nvars); + template + void hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, + amrex::MultiFab &x1FaceVel, amrex::MultiFab const &x1Flat, amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, + int ng_reconstruct, int nvars); - template - void hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, - amrex::MultiFab &x1FaceVel, int ng_reconstruct, int nvars); + template + void hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, amrex::MultiFab &x1Flux, + amrex::MultiFab &x1FaceVel, int ng_reconstruct, int nvars); - void replaceFluxes(std::array &fluxes, std::array &FOfluxes, - amrex::iMultiFab &redoFlag); + void replaceFluxes(std::array &fluxes, std::array &FOfluxes, + amrex::iMultiFab &redoFlag); }; template void RadhydroSimulation::defineComponentNames() { - // cell-centred - // add hydro state variables - if constexpr (Physics_Traits::is_hydro_enabled || Physics_Traits::is_radiation_enabled) { - std::vector hydroNames = {"gasDensity", "x-GasMomentum", "y-GasMomentum", "z-GasMomentum", "gasEnergy", "gasInternalEnergy"}; - componentNames_cc_.insert(componentNames_cc_.end(), hydroNames.begin(), hydroNames.end()); - } - // add passive scalar variables - if constexpr (Physics_Traits::numPassiveScalars > 0) { - std::vector scalarNames = getScalarVariableNames(); - componentNames_cc_.insert(componentNames_cc_.end(), scalarNames.begin(), scalarNames.end()); - } - // add radiation state variables - if constexpr (Physics_Traits::is_radiation_enabled) { - std::vector radNames = {}; - for (int i = 0; i < Physics_Traits::nGroups; ++i) { - radNames.push_back("radEnergy-Group" + std::to_string(i)); - radNames.push_back("x-RadFlux-Group" + std::to_string(i)); - radNames.push_back("y-RadFlux-Group" + std::to_string(i)); - radNames.push_back("z-RadFlux-Group" + std::to_string(i)); - } - componentNames_cc_.insert(componentNames_cc_.end(), radNames.begin(), radNames.end()); - } - - // face-centred - - // add face-centered velocities - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-velocity"}); - } - // add mhd state variables - if constexpr (Physics_Traits::is_mhd_enabled) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-BField"}); - } - } + // cell-centred + // add hydro state variables + if constexpr (Physics_Traits::is_hydro_enabled || Physics_Traits::is_radiation_enabled) { + std::vector hydroNames = {"gasDensity", "x-GasMomentum", "y-GasMomentum", "z-GasMomentum", "gasEnergy", "gasInternalEnergy"}; + componentNames_cc_.insert(componentNames_cc_.end(), hydroNames.begin(), hydroNames.end()); + } + // add passive scalar variables + if constexpr (Physics_Traits::numPassiveScalars > 0) { + std::vector scalarNames = getScalarVariableNames(); + componentNames_cc_.insert(componentNames_cc_.end(), scalarNames.begin(), scalarNames.end()); + } + // add radiation state variables + if constexpr (Physics_Traits::is_radiation_enabled) { + std::vector radNames = {}; + for (int i = 0; i < Physics_Traits::nGroups; ++i) { + radNames.push_back("radEnergy-Group" + std::to_string(i)); + radNames.push_back("x-RadFlux-Group" + std::to_string(i)); + radNames.push_back("y-RadFlux-Group" + std::to_string(i)); + radNames.push_back("z-RadFlux-Group" + std::to_string(i)); + } + componentNames_cc_.insert(componentNames_cc_.end(), radNames.begin(), radNames.end()); + } + + // face-centred + + // add face-centered velocities + for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-velocity"}); + } + // add mhd state variables + if constexpr (Physics_Traits::is_mhd_enabled) { + for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + componentNames_fc_.push_back({quokka::face_dir_str[idim] + "-BField"}); + } + } } template auto RadhydroSimulation::getScalarVariableNames() -> std::vector { - // return vector of names for the passive scalars - // this can be specialized by the user to provide more descriptive names - // (these names are used to label the variables in the plotfiles) - - std::vector names; - int nscalars = HydroSystem::nscalars_; - names.reserve(nscalars); - for (int n = 0; n < nscalars; ++n) { - // write string 'scalar_1', etc. - names.push_back(fmt::format("scalar_{}", n)); - } - return names; + // return vector of names for the passive scalars + // this can be specialized by the user to provide more descriptive names + // (these names are used to label the variables in the plotfiles) + + std::vector names; + int nscalars = HydroSystem::nscalars_; + names.reserve(nscalars); + for (int n = 0; n < nscalars; ++n) { + // write string 'scalar_1', etc. + names.push_back(fmt::format("scalar_{}", n)); + } + return names; } template void RadhydroSimulation::readParmParse() { - // set hydro runtime parameters - { - amrex::ParmParse hpp("hydro"); - hpp.query("low_level_debugging_output", lowLevelDebuggingOutput_); - hpp.query("rk_integrator_order", integratorOrder_); - hpp.query("reconstruction_order", reconstructionOrder_); - hpp.query("use_dual_energy", useDualEnergy_); - hpp.query("abort_on_fofc_failure", abortOnFofcFailure_); - hpp.query("artificial_viscosity_coefficient", artificialViscosityK_); - } - - // set cooling runtime parameters - { - amrex::ParmParse hpp("cooling"); - int alwaysReadTables = 0; - hpp.query("enabled", enableCooling_); - hpp.query("read_tables_even_if_disabled", alwaysReadTables); - hpp.query("grackle_data_file", coolingTableFilename_); - - if ((enableCooling_ == 1) || (alwaysReadTables == 1)) { - // read Cloudy tables - amrex::Print() << "Reading Cloudy tables...\n"; - quokka::cooling::readCloudyData(coolingTableFilename_, cloudyTables_); - } - } + // set hydro runtime parameters + { + amrex::ParmParse hpp("hydro"); + hpp.query("low_level_debugging_output", lowLevelDebuggingOutput_); + hpp.query("rk_integrator_order", integratorOrder_); + hpp.query("reconstruction_order", reconstructionOrder_); + hpp.query("use_dual_energy", useDualEnergy_); + hpp.query("abort_on_fofc_failure", abortOnFofcFailure_); + hpp.query("artificial_viscosity_coefficient", artificialViscosityK_); + } + + // set cooling runtime parameters + { + amrex::ParmParse hpp("cooling"); + int alwaysReadTables = 0; + hpp.query("enabled", enableCooling_); + hpp.query("read_tables_even_if_disabled", alwaysReadTables); + hpp.query("grackle_data_file", coolingTableFilename_); + + if ((enableCooling_ == 1) || (alwaysReadTables == 1)) { + // read Cloudy tables + amrex::Print() << "Reading Cloudy tables...\n"; + quokka::cooling::readCloudyData(coolingTableFilename_, cloudyTables_); + } + } #ifdef PRIMORDIAL_CHEM - // set chemistry runtime parameters - { - amrex::ParmParse hpp("primordial_chem"); - hpp.query("enabled", enableChemistry_); - hpp.query("max_density_allowed", max_density_allowed); // chemistry is not accurate for densities > 3e-6 - hpp.query("min_density_allowed", min_density_allowed); // don't do chemistry in cells with densities below the minimum density specified - } + // set chemistry runtime parameters + { + amrex::ParmParse hpp("primordial_chem"); + hpp.query("enabled", enableChemistry_); + hpp.query("max_density_allowed", max_density_allowed); // chemistry is not accurate for densities > 3e-6 + hpp.query("min_density_allowed", min_density_allowed); // don't do chemistry in cells with densities below the minimum density specified + } #endif - // set radiation runtime parameters - { - amrex::ParmParse rpp("radiation"); - rpp.query("reconstruction_order", radiationReconstructionOrder_); - rpp.query("cfl", radiationCflNumber_); - } + // set radiation runtime parameters + { + amrex::ParmParse rpp("radiation"); + rpp.query("reconstruction_order", radiationReconstructionOrder_); + rpp.query("cfl", radiationCflNumber_); + } } template auto RadhydroSimulation::computeNumberOfRadiationSubsteps(int lev, amrex::Real dt_lev_hydro) -> int { - // compute radiation timestep - auto const &dx = geom[lev].CellSizeArray(); - amrex::Real c_hat = RadSystem::c_hat_; - amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); - amrex::Real dtrad_tmp = radiationCflNumber_ * (dx_min / c_hat); - int nsubSteps = std::ceil(dt_lev_hydro / dtrad_tmp); - return nsubSteps; + // compute radiation timestep + auto const &dx = geom[lev].CellSizeArray(); + amrex::Real c_hat = RadSystem::c_hat_; + amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); + amrex::Real dtrad_tmp = radiationCflNumber_ * (dx_min / c_hat); + int nsubSteps = std::ceil(dt_lev_hydro / dtrad_tmp); + return nsubSteps; } template void RadhydroSimulation::computeMaxSignalLocal(int const level) { - BL_PROFILE("RadhydroSimulation::computeMaxSignalLocal()"); - - // hydro: loop over local grids, compute CFL timestep - for (amrex::MFIter iter(state_new_cc_[level]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[level].const_array(iter); - auto const &maxSignal = max_signal_speed_[level].array(iter); - - if constexpr (Physics_Traits::is_hydro_enabled && !(Physics_Traits::is_radiation_enabled)) { - // hydro only - HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); - } else if constexpr (Physics_Traits::is_radiation_enabled) { - // radiation hydro, or radiation only - RadSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); - if constexpr (Physics_Traits::is_hydro_enabled) { - auto maxSignalHydroFAB = amrex::FArrayBox(indexRange); - auto const &maxSignalHydro = maxSignalHydroFAB.array(); - HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignalHydro, indexRange); - const int maxSubsteps = maxSubsteps_; - // ensure that we use the smaller of the two timesteps - amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real const maxSignalRadiation = maxSignal(i, j, k) / static_cast(maxSubsteps); - maxSignal(i, j, k) = std::max(maxSignalRadiation, maxSignalHydro(i, j, k)); - }); - } - } else { - // no physics modules enabled, why are we running? - amrex::Abort("At least one of hydro or radiation must be enabled! Cannot " - "compute a time step."); - } - } + BL_PROFILE("RadhydroSimulation::computeMaxSignalLocal()"); + + // hydro: loop over local grids, compute CFL timestep + for (amrex::MFIter iter(state_new_cc_[level]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateNew = state_new_cc_[level].const_array(iter); + auto const &maxSignal = max_signal_speed_[level].array(iter); + + if constexpr (Physics_Traits::is_hydro_enabled && !(Physics_Traits::is_radiation_enabled)) { + // hydro only + HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); + } else if constexpr (Physics_Traits::is_radiation_enabled) { + // radiation hydro, or radiation only + RadSystem::ComputeMaxSignalSpeed(stateNew, maxSignal, indexRange); + if constexpr (Physics_Traits::is_hydro_enabled) { + auto maxSignalHydroFAB = amrex::FArrayBox(indexRange); + auto const &maxSignalHydro = maxSignalHydroFAB.array(); + HydroSystem::ComputeMaxSignalSpeed(stateNew, maxSignalHydro, indexRange); + const int maxSubsteps = maxSubsteps_; + // ensure that we use the smaller of the two timesteps + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real const maxSignalRadiation = maxSignal(i, j, k) / static_cast(maxSubsteps); + maxSignal(i, j, k) = std::max(maxSignalRadiation, maxSignalHydro(i, j, k)); + }); + } + } else { + // no physics modules enabled, why are we running? + amrex::Abort("At least one of hydro or radiation must be enabled! Cannot " + "compute a time step."); + } + } } template auto RadhydroSimulation::computeExtraPhysicsTimestep(int const level) -> amrex::Real { - BL_PROFILE("RadhydroSimulation::computeExtraPhysicsTimestep()"); - // users can override this to enforce additional timestep constraints - return std::numeric_limits::max(); + BL_PROFILE("RadhydroSimulation::computeExtraPhysicsTimestep()"); + // users can override this to enforce additional timestep constraints + return std::numeric_limits::max(); } #if !defined(NDEBUG) @@ -450,300 +450,300 @@ template auto RadhydroSimulation::computeExtraPh template void RadhydroSimulation::checkHydroStates(amrex::MultiFab &mf, char const *file, int line) { - BL_PROFILE("RadhydroSimulation::checkHydroStates()"); - - bool validStates = HydroSystem::CheckStatesValid(mf); - amrex::ParallelDescriptor::ReduceBoolAnd(validStates); - - if (!validStates) { - amrex::Print() << "Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")\n"; - amrex::Print() << "Writing checkpoint for debugging...\n"; - amrex::MFIter::allowMultipleMFIters(true); - WriteCheckpointFile(); - amrex::Abort("Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")"); - } + BL_PROFILE("RadhydroSimulation::checkHydroStates()"); + + bool validStates = HydroSystem::CheckStatesValid(mf); + amrex::ParallelDescriptor::ReduceBoolAnd(validStates); + + if (!validStates) { + amrex::Print() << "Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")\n"; + amrex::Print() << "Writing checkpoint for debugging...\n"; + amrex::MFIter::allowMultipleMFIters(true); + WriteCheckpointFile(); + amrex::Abort("Hydro states invalid (" + std::string(file) + ":" + std::to_string(line) + ")"); + } } template void RadhydroSimulation::preCalculateInitialConditions() { - // default empty implementation - // user should implement using problem-specific template specialization + // default empty implementation + // user should implement using problem-specific template specialization } template void RadhydroSimulation::setInitialConditionsOnGrid(quokka::grid grid_elem) { - // default empty implementation - // user should implement using problem-specific template specialization + // default empty implementation + // user should implement using problem-specific template specialization } template void RadhydroSimulation::setInitialConditionsOnGridFaceVars(quokka::grid grid_elem) { - // default empty implementation - // user should implement using problem-specific template specialization - // note: an implementation is only required if face-centered vars are used + // default empty implementation + // user should implement using problem-specific template specialization + // note: an implementation is only required if face-centered vars are used } template void RadhydroSimulation::createInitialParticles() { - // default empty implementation - // user should implement using problem-specific template specialization - // note: an implementation is only required if particles are used + // default empty implementation + // user should implement using problem-specific template specialization + // note: an implementation is only required if particles are used } template void RadhydroSimulation::computeAfterTimestep() { - // do nothing -- user should implement if desired + // do nothing -- user should implement if desired } template void RadhydroSimulation::computeAfterLevelAdvance(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) { - // user should implement if desired + // user should implement if desired } template void RadhydroSimulation::addStrangSplitSources(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt) { - // user should implement - // (when Strang splitting is enabled, dt is actually 0.5*dt_lev) + // user should implement + // (when Strang splitting is enabled, dt is actually 0.5*dt_lev) } template void RadhydroSimulation::addStrangSplitSourcesWithBuiltin(amrex::MultiFab &state, int lev, amrex::Real time, amrex::Real dt) { - if (enableCooling_ == 1) { - // compute cooling - quokka::cooling::computeCooling(state, dt, cloudyTables_, tempFloor_); - } + if (enableCooling_ == 1) { + // compute cooling + quokka::cooling::computeCooling(state, dt, cloudyTables_, tempFloor_); + } #ifdef PRIMORDIAL_CHEM - if (enableChemistry_ == 1) { - // compute chemistry - quokka::chemistry::computeChemistry(state, dt, max_density_allowed, min_density_allowed); - } + if (enableChemistry_ == 1) { + // compute chemistry + quokka::chemistry::computeChemistry(state, dt, max_density_allowed, min_density_allowed); + } #endif - // compute user-specified sources - addStrangSplitSources(state, lev, time, dt); + // compute user-specified sources + addStrangSplitSources(state, lev, time, dt); } template void RadhydroSimulation::ComputeDerivedVar(int lev, std::string const &dname, amrex::MultiFab &mf, const int ncomp) const { - // compute derived variables and save in 'mf' -- user should implement + // compute derived variables and save in 'mf' -- user should implement } template auto RadhydroSimulation::ComputeProjections(int /*dir*/) const -> std::unordered_map> { - // compute projections and return as unordered_map -- user should implement - return std::unordered_map>{}; + // compute projections and return as unordered_map -- user should implement + return std::unordered_map>{}; } template auto RadhydroSimulation::ComputeStatistics() -> std::map { - // compute statistics and return a std::map -- user should implement - // IMPORTANT: the user is responsible for performing any necessary MPI reductions before returning - return std::map{}; + // compute statistics and return a std::map -- user should implement + // IMPORTANT: the user is responsible for performing any necessary MPI reductions before returning + return std::map{}; } template void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) { - // tag cells for refinement -- user should implement + // tag cells for refinement -- user should implement } template void RadhydroSimulation::computeReferenceSolution(amrex::MultiFab &ref, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo) + amrex::GpuArray const &prob_lo) { - // user should implement + // user should implement } template void RadhydroSimulation::computeAfterEvolve(amrex::Vector &initSumCons) { - amrex::GpuArray const &dx0 = geom[0].CellSizeArray(); - amrex::Real const vol = AMREX_D_TERM(dx0[0], *dx0[1], *dx0[2]); - - // check conservation of total energy - amrex::Real const Egas0 = initSumCons[RadSystem::gasEnergy_index]; - amrex::Real const Egas = state_new_cc_[0].sum(RadSystem::gasEnergy_index) * vol; - - amrex::Real Etot0 = NAN; - amrex::Real Etot = NAN; - if constexpr (Physics_Traits::is_radiation_enabled) { - amrex::Real Erad0 = 0.; - for (int g = 0; g < Physics_Traits::nGroups; ++g) { - Erad0 += initSumCons[RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g]; - } - Etot0 = Egas0 + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad0; - amrex::Real Erad = 0.; - for (int g = 0; g < Physics_Traits::nGroups; ++g) { - Erad += state_new_cc_[0].sum(RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g) * vol; - } - Etot = Egas + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad; - } else { - Etot0 = Egas0; - Etot = Egas; - } - - amrex::Real const abs_err = (Etot - Etot0); - amrex::Real const rel_err = abs_err / Etot0; - - amrex::Print() << "\nInitial gas+radiation energy = " << Etot0 << std::endl; - amrex::Print() << "Final gas+radiation energy = " << Etot << std::endl; - amrex::Print() << "\tabsolute conservation error = " << abs_err << std::endl; - amrex::Print() << "\trelative conservation error = " << rel_err << std::endl; - amrex::Print() << std::endl; - - if (computeReferenceSolution_) { - // compute reference solution - const int ncomp = state_new_cc_[0].nComp(); - const int nghost = state_new_cc_[0].nGrow(); - amrex::MultiFab state_ref_level0(boxArray(0), DistributionMap(0), ncomp, nghost); - computeReferenceSolution(state_ref_level0, geom[0].CellSizeArray(), geom[0].ProbLoArray()); - - // compute error norm - amrex::MultiFab residual(boxArray(0), DistributionMap(0), ncomp, nghost); - amrex::MultiFab::Copy(residual, state_ref_level0, 0, 0, ncomp, nghost); - amrex::MultiFab::Saxpy(residual, -1., state_new_cc_[0], 0, 0, ncomp, nghost); - - amrex::Real sol_norm = 0.; - amrex::Real err_norm = 0.; - // compute rms of each component - for (int n = 0; n < ncomp; ++n) { - sol_norm += std::pow(state_ref_level0.norm1(n), 2); - err_norm += std::pow(residual.norm1(n), 2); - } - sol_norm = std::sqrt(sol_norm); - err_norm = std::sqrt(err_norm); - - const double rel_error = err_norm / sol_norm; - errorNorm_ = rel_error; - amrex::Print() << "Relative rms L1 error norm = " << rel_error << std::endl; - } - amrex::Print() << std::endl; - - // compute average number of radiation subcycles per timestep - double const avg_rad_subcycles = static_cast(radiationCellUpdates_) / static_cast(cellUpdates_); - amrex::Print() << "avg. num. of radiation subcycles = " << avg_rad_subcycles << std::endl; - amrex::Print() << std::endl; + amrex::GpuArray const &dx0 = geom[0].CellSizeArray(); + amrex::Real const vol = AMREX_D_TERM(dx0[0], *dx0[1], *dx0[2]); + + // check conservation of total energy + amrex::Real const Egas0 = initSumCons[RadSystem::gasEnergy_index]; + amrex::Real const Egas = state_new_cc_[0].sum(RadSystem::gasEnergy_index) * vol; + + amrex::Real Etot0 = NAN; + amrex::Real Etot = NAN; + if constexpr (Physics_Traits::is_radiation_enabled) { + amrex::Real Erad0 = 0.; + for (int g = 0; g < Physics_Traits::nGroups; ++g) { + Erad0 += initSumCons[RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g]; + } + Etot0 = Egas0 + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad0; + amrex::Real Erad = 0.; + for (int g = 0; g < Physics_Traits::nGroups; ++g) { + Erad += state_new_cc_[0].sum(RadSystem::radEnergy_index + Physics_NumVars::numRadVars * g) * vol; + } + Etot = Egas + (RadSystem::c_light_ / RadSystem::c_hat_) * Erad; + } else { + Etot0 = Egas0; + Etot = Egas; + } + + amrex::Real const abs_err = (Etot - Etot0); + amrex::Real const rel_err = abs_err / Etot0; + + amrex::Print() << "\nInitial gas+radiation energy = " << Etot0 << std::endl; + amrex::Print() << "Final gas+radiation energy = " << Etot << std::endl; + amrex::Print() << "\tabsolute conservation error = " << abs_err << std::endl; + amrex::Print() << "\trelative conservation error = " << rel_err << std::endl; + amrex::Print() << std::endl; + + if (computeReferenceSolution_) { + // compute reference solution + const int ncomp = state_new_cc_[0].nComp(); + const int nghost = state_new_cc_[0].nGrow(); + amrex::MultiFab state_ref_level0(boxArray(0), DistributionMap(0), ncomp, nghost); + computeReferenceSolution(state_ref_level0, geom[0].CellSizeArray(), geom[0].ProbLoArray()); + + // compute error norm + amrex::MultiFab residual(boxArray(0), DistributionMap(0), ncomp, nghost); + amrex::MultiFab::Copy(residual, state_ref_level0, 0, 0, ncomp, nghost); + amrex::MultiFab::Saxpy(residual, -1., state_new_cc_[0], 0, 0, ncomp, nghost); + + amrex::Real sol_norm = 0.; + amrex::Real err_norm = 0.; + // compute rms of each component + for (int n = 0; n < ncomp; ++n) { + sol_norm += std::pow(state_ref_level0.norm1(n), 2); + err_norm += std::pow(residual.norm1(n), 2); + } + sol_norm = std::sqrt(sol_norm); + err_norm = std::sqrt(err_norm); + + const double rel_error = err_norm / sol_norm; + errorNorm_ = rel_error; + amrex::Print() << "Relative rms L1 error norm = " << rel_error << std::endl; + } + amrex::Print() << std::endl; + + // compute average number of radiation subcycles per timestep + double const avg_rad_subcycles = static_cast(radiationCellUpdates_) / static_cast(cellUpdates_); + amrex::Print() << "avg. num. of radiation subcycles = " << avg_rad_subcycles << std::endl; + amrex::Print() << std::endl; } template void RadhydroSimulation::advanceSingleTimestepAtLevel(int lev, amrex::Real time, amrex::Real dt_lev, int ncycle) { - BL_PROFILE("RadhydroSimulation::advanceSingleTimestepAtLevel()"); - - // get flux registers - amrex::YAFluxRegister *fr_as_crse = nullptr; - amrex::YAFluxRegister *fr_as_fine = nullptr; - if (do_reflux != 0) { - if (lev < finestLevel()) { - fr_as_crse = flux_reg_[lev + 1].get(); - fr_as_crse->reset(); - } - if (lev > 0) { - fr_as_fine = flux_reg_[lev].get(); - } - } - - // since we are starting a new timestep, need to swap old and new state vectors - std::swap(state_old_cc_[lev], state_new_cc_[lev]); - if (Physics_Indices::nvarTotal_fc > 0) { - std::swap(state_old_fc_[lev], state_new_fc_[lev]); - } - - // check hydro states before update (this can be caused by the flux register!) - CHECK_HYDRO_STATES(state_old_cc_[lev]); - - // advance hydro - if constexpr (Physics_Traits::is_hydro_enabled) { - advanceHydroAtLevelWithRetries(lev, time, dt_lev, fr_as_crse, fr_as_fine); - } else { - // copy hydro vars from state_old_cc_ to state_new_cc_ - // (otherwise radiation update will be wrong!) - amrex::MultiFab::Copy(state_new_cc_[lev], state_old_cc_[lev], 0, 0, ncompHydro_, 0); - } - - // check hydro states after hydro update - CHECK_HYDRO_STATES(state_new_cc_[lev]); - - // subcycle radiation - if constexpr (Physics_Traits::is_radiation_enabled) { - subcycleRadiationAtLevel(lev, time, dt_lev, fr_as_crse, fr_as_fine); - } - - // check hydro states after radiation update - CHECK_HYDRO_STATES(state_new_cc_[lev]); - - // compute any operator-split terms here (user-defined) - computeAfterLevelAdvance(lev, time, dt_lev, ncycle); - - // check hydro states after user work - CHECK_HYDRO_STATES(state_new_cc_[lev]); - - // check state validity - AMREX_ASSERT(!state_new_cc_[lev].contains_nan(0, state_new_cc_[lev].nComp())); + BL_PROFILE("RadhydroSimulation::advanceSingleTimestepAtLevel()"); + + // get flux registers + amrex::YAFluxRegister *fr_as_crse = nullptr; + amrex::YAFluxRegister *fr_as_fine = nullptr; + if (do_reflux != 0) { + if (lev < finestLevel()) { + fr_as_crse = flux_reg_[lev + 1].get(); + fr_as_crse->reset(); + } + if (lev > 0) { + fr_as_fine = flux_reg_[lev].get(); + } + } + + // since we are starting a new timestep, need to swap old and new state vectors + std::swap(state_old_cc_[lev], state_new_cc_[lev]); + if (Physics_Indices::nvarTotal_fc > 0) { + std::swap(state_old_fc_[lev], state_new_fc_[lev]); + } + + // check hydro states before update (this can be caused by the flux register!) + CHECK_HYDRO_STATES(state_old_cc_[lev]); + + // advance hydro + if constexpr (Physics_Traits::is_hydro_enabled) { + advanceHydroAtLevelWithRetries(lev, time, dt_lev, fr_as_crse, fr_as_fine); + } else { + // copy hydro vars from state_old_cc_ to state_new_cc_ + // (otherwise radiation update will be wrong!) + amrex::MultiFab::Copy(state_new_cc_[lev], state_old_cc_[lev], 0, 0, ncompHydro_, 0); + } + + // check hydro states after hydro update + CHECK_HYDRO_STATES(state_new_cc_[lev]); + + // subcycle radiation + if constexpr (Physics_Traits::is_radiation_enabled) { + subcycleRadiationAtLevel(lev, time, dt_lev, fr_as_crse, fr_as_fine); + } + + // check hydro states after radiation update + CHECK_HYDRO_STATES(state_new_cc_[lev]); + + // compute any operator-split terms here (user-defined) + computeAfterLevelAdvance(lev, time, dt_lev, ncycle); + + // check hydro states after user work + CHECK_HYDRO_STATES(state_new_cc_[lev]); + + // check state validity + AMREX_ASSERT(!state_new_cc_[lev].contains_nan(0, state_new_cc_[lev].nComp())); } template void RadhydroSimulation::fillPoissonRhsAtLevel(amrex::MultiFab &rhs_mf, const int lev) { - // add hydro density to Poisson rhs - auto const &state = state_new_cc_[lev].const_arrays(); - auto rhs = rhs_mf.arrays(); - const Real G = Gconst_; - - amrex::ParallelFor(rhs_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - // *add* density to rhs_mf - // (N.B. particles **will not work** if you overwrite the density here!) - rhs[bx](i, j, k) += 4.0 * M_PI * G * state[bx](i, j, k, HydroSystem::density_index); - }); - amrex::Gpu::streamSynchronizeAll(); + // add hydro density to Poisson rhs + auto const &state = state_new_cc_[lev].const_arrays(); + auto rhs = rhs_mf.arrays(); + const Real G = Gconst_; + + amrex::ParallelFor(rhs_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + // *add* density to rhs_mf + // (N.B. particles **will not work** if you overwrite the density here!) + rhs[bx](i, j, k) += 4.0 * M_PI * G * state[bx](i, j, k, HydroSystem::density_index); + }); + amrex::Gpu::streamSynchronizeAll(); } template void RadhydroSimulation::applyPoissonGravityAtLevel(amrex::MultiFab const &phi_mf, const int lev, const amrex::Real dt) { - if constexpr (AMREX_SPACEDIM == 3) { - // apply Poisson gravity operator on level 'lev' - auto const &dx = geom[lev].CellSizeArray(); - auto const &phi = phi_mf.const_arrays(); - auto state = state_new_cc_[lev].arrays(); - - amrex::ParallelFor(phi_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - // add operator-split gravitational acceleration - const amrex::Real rho = state[bx](i, j, k, HydroSystem::density_index); - amrex::Real px = state[bx](i, j, k, HydroSystem::x1Momentum_index); - amrex::Real py = state[bx](i, j, k, HydroSystem::x2Momentum_index); - amrex::Real pz = state[bx](i, j, k, HydroSystem::x3Momentum_index); - const amrex::Real KE_old = 0.5 * (px * px + py * py + pz * pz) / rho; - - // g = -grad \phi - amrex::Real gx = -0.5 * (phi[bx](i + 1, j, k) - phi[bx](i - 1, j, k)) / dx[0]; - amrex::Real gy = -0.5 * (phi[bx](i, j + 1, k) - phi[bx](i, j - 1, k)) / dx[1]; - amrex::Real gz = -0.5 * (phi[bx](i, j, k + 1) - phi[bx](i, j, k - 1)) / dx[2]; - - px += dt * rho * gx; - py += dt * rho * gy; - pz += dt * rho * gz; - const amrex::Real KE_new = 0.5 * (px * px + py * py + pz * pz) / rho; - const amrex::Real dKE = KE_new - KE_old; - - state[bx](i, j, k, HydroSystem::x1Momentum_index) = px; - state[bx](i, j, k, HydroSystem::x2Momentum_index) = py; - state[bx](i, j, k, HydroSystem::x3Momentum_index) = pz; - state[bx](i, j, k, HydroSystem::energy_index) += dKE; - }); - } + if constexpr (AMREX_SPACEDIM == 3) { + // apply Poisson gravity operator on level 'lev' + auto const &dx = geom[lev].CellSizeArray(); + auto const &phi = phi_mf.const_arrays(); + auto state = state_new_cc_[lev].arrays(); + + amrex::ParallelFor(phi_mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + // add operator-split gravitational acceleration + const amrex::Real rho = state[bx](i, j, k, HydroSystem::density_index); + amrex::Real px = state[bx](i, j, k, HydroSystem::x1Momentum_index); + amrex::Real py = state[bx](i, j, k, HydroSystem::x2Momentum_index); + amrex::Real pz = state[bx](i, j, k, HydroSystem::x3Momentum_index); + const amrex::Real KE_old = 0.5 * (px * px + py * py + pz * pz) / rho; + + // g = -grad \phi + amrex::Real gx = -0.5 * (phi[bx](i + 1, j, k) - phi[bx](i - 1, j, k)) / dx[0]; + amrex::Real gy = -0.5 * (phi[bx](i, j + 1, k) - phi[bx](i, j - 1, k)) / dx[1]; + amrex::Real gz = -0.5 * (phi[bx](i, j, k + 1) - phi[bx](i, j, k - 1)) / dx[2]; + + px += dt * rho * gx; + py += dt * rho * gy; + pz += dt * rho * gz; + const amrex::Real KE_new = 0.5 * (px * px + py * py + pz * pz) / rho; + const amrex::Real dKE = KE_new - KE_old; + + state[bx](i, j, k, HydroSystem::x1Momentum_index) = px; + state[bx](i, j, k, HydroSystem::x2Momentum_index) = py; + state[bx](i, j, k, HydroSystem::x3Momentum_index) = pz; + state[bx](i, j, k, HydroSystem::energy_index) += dKE; + }); + } } // fix-up any unphysical states created by AMR operations // (e.g., caused by the flux register or from interpolation) template void RadhydroSimulation::FixupState(int lev) { - BL_PROFILE("RadhydroSimulation::FixupState()"); + BL_PROFILE("RadhydroSimulation::FixupState()"); - // fix hydro state - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); - // sync internal energy and total energy - HydroSystem::SyncDualEnergy(state_new_cc_[lev]); + // fix hydro state + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_new_cc_[lev]); + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, state_old_cc_[lev]); + // sync internal energy and total energy + HydroSystem::SyncDualEnergy(state_new_cc_[lev]); } // Compute a new multifab 'mf' by copying in state from valid region and filling @@ -752,1110 +752,1110 @@ template void RadhydroSimulation::FixupState(int // are implemented in this class (and must be *static* functions). template void RadhydroSimulation::FillPatch(int lev, amrex::Real time, amrex::MultiFab &mf, int icomp, int ncomp, quokka::centering cen, - quokka::direction dir, FillPatchType fptype) + quokka::direction dir, FillPatchType fptype) { - BL_PROFILE("AMRSimulation::FillPatch()"); - - amrex::Vector cmf; - amrex::Vector fmf; - amrex::Vector ctime; - amrex::Vector ftime; - - if (lev == 0) { - // in this case, should return either state_new_[lev] or state_old_[lev] - GetData(lev, time, fmf, ftime, cen, dir); - } else { - // in this case, should return either state_new_[lev] or state_old_[lev] - // returns old state, new state, or both depending on 'time' - GetData(lev, time, fmf, ftime, cen, dir); - GetData(lev - 1, time, cmf, ctime, cen, dir); - } - - if (cen == quokka::centering::cc) { - FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_cc_, cen, fptype, PreInterpState, PostInterpState); - } else if (cen == quokka::centering::fc) { - FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_fc_, cen, fptype, PreInterpState, PostInterpState); - } + BL_PROFILE("AMRSimulation::FillPatch()"); + + amrex::Vector cmf; + amrex::Vector fmf; + amrex::Vector ctime; + amrex::Vector ftime; + + if (lev == 0) { + // in this case, should return either state_new_[lev] or state_old_[lev] + GetData(lev, time, fmf, ftime, cen, dir); + } else { + // in this case, should return either state_new_[lev] or state_old_[lev] + // returns old state, new state, or both depending on 'time' + GetData(lev, time, fmf, ftime, cen, dir); + GetData(lev - 1, time, cmf, ctime, cen, dir); + } + + if (cen == quokka::centering::cc) { + FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_cc_, cen, fptype, PreInterpState, PostInterpState); + } else if (cen == quokka::centering::fc) { + FillPatchWithData(lev, time, mf, cmf, ctime, fmf, ftime, icomp, ncomp, BCs_fc_, cen, fptype, PreInterpState, PostInterpState); + } } template void RadhydroSimulation::PreInterpState(amrex::MultiFab &mf, int scomp, int ncomp) { - BL_PROFILE("RadhydroSimulation::PreInterpState()"); - - auto const &cons = mf.arrays(); - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { - const auto rho = cons[bx](i, j, k, HydroSystem::density_index); - const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); - const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); - const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); - const auto Etot = cons[bx](i, j, k, HydroSystem::energy_index); - const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); - - // replace hydro total energy with specific internal energy (SIE) - const auto e = (Etot - kinetic_energy) / rho; - cons[bx](i, j, k, HydroSystem::energy_index) = e; - }); + BL_PROFILE("RadhydroSimulation::PreInterpState()"); + + auto const &cons = mf.arrays(); + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { + const auto rho = cons[bx](i, j, k, HydroSystem::density_index); + const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); + const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); + const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); + const auto Etot = cons[bx](i, j, k, HydroSystem::energy_index); + const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); + + // replace hydro total energy with specific internal energy (SIE) + const auto e = (Etot - kinetic_energy) / rho; + cons[bx](i, j, k, HydroSystem::energy_index) = e; + }); } template void RadhydroSimulation::PostInterpState(amrex::MultiFab &mf, int scomp, int ncomp) { - BL_PROFILE("RadhydroSimulation::PostInterpState()"); - - auto const &cons = mf.arrays(); - amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { - const auto rho = cons[bx](i, j, k, HydroSystem::density_index); - const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); - const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); - const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); - const auto e = cons[bx](i, j, k, HydroSystem::energy_index); - const auto Eint = rho * e; - const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); - - // recompute hydro total energy from Eint + KE - const auto Etot = Eint + kinetic_energy; - cons[bx](i, j, k, HydroSystem::energy_index) = Etot; - }); + BL_PROFILE("RadhydroSimulation::PostInterpState()"); + + auto const &cons = mf.arrays(); + amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) { + const auto rho = cons[bx](i, j, k, HydroSystem::density_index); + const auto px = cons[bx](i, j, k, HydroSystem::x1Momentum_index); + const auto py = cons[bx](i, j, k, HydroSystem::x2Momentum_index); + const auto pz = cons[bx](i, j, k, HydroSystem::x3Momentum_index); + const auto e = cons[bx](i, j, k, HydroSystem::energy_index); + const auto Eint = rho * e; + const auto kinetic_energy = (px * px + py * py + pz * pz) / (2.0 * rho); + + // recompute hydro total energy from Eint + KE + const auto Etot = Eint + kinetic_energy; + cons[bx](i, j, k, HydroSystem::energy_index) = Etot; + }); } template template auto RadhydroSimulation::computeAxisAlignedProfile(const int axis, F const &user_f) -> amrex::Gpu::HostVector { - // compute a 1D profile of user_f(i, j, k, state) along the given axis. - BL_PROFILE("RadhydroSimulation::computeAxisAlignedProfile()"); - - // allocate temporary multifabs - amrex::Vector q; - q.resize(finest_level + 1); - for (int lev = 0; lev <= finest_level; ++lev) { - q[lev].define(boxArray(lev), DistributionMap(lev), 1, 0); - } - - // evaluate user_f on all levels - for (int lev = 0; lev <= finest_level; ++lev) { - for (amrex::MFIter iter(q[lev]); iter.isValid(); ++iter) { - auto const &box = iter.validbox(); - auto const &state = state_new_cc_[lev].const_array(iter); - auto const &result = q[lev].array(iter); - amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { result(i, j, k) = user_f(i, j, k, state); }); - } - } - - // average down - for (int crse_lev = finest_level - 1; crse_lev >= 0; --crse_lev) { - amrex::average_down(q[crse_lev + 1], q[crse_lev], geom[crse_lev + 1], geom[crse_lev], 0, q[crse_lev].nComp(), refRatio(crse_lev)); - } - - // compute 1D profile from level 0 multifab - amrex::Box domain = geom[0].Domain(); - auto profile = amrex::sumToLine(q[0], 0, q[0].nComp(), domain, axis); - - // normalize profile - amrex::Long numCells = domain.numPts() / domain.length(axis); - for (double &bin : profile) { - bin /= static_cast(numCells); - } - - return profile; + // compute a 1D profile of user_f(i, j, k, state) along the given axis. + BL_PROFILE("RadhydroSimulation::computeAxisAlignedProfile()"); + + // allocate temporary multifabs + amrex::Vector q; + q.resize(finest_level + 1); + for (int lev = 0; lev <= finest_level; ++lev) { + q[lev].define(boxArray(lev), DistributionMap(lev), 1, 0); + } + + // evaluate user_f on all levels + for (int lev = 0; lev <= finest_level; ++lev) { + for (amrex::MFIter iter(q[lev]); iter.isValid(); ++iter) { + auto const &box = iter.validbox(); + auto const &state = state_new_cc_[lev].const_array(iter); + auto const &result = q[lev].array(iter); + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { result(i, j, k) = user_f(i, j, k, state); }); + } + } + + // average down + for (int crse_lev = finest_level - 1; crse_lev >= 0; --crse_lev) { + amrex::average_down(q[crse_lev + 1], q[crse_lev], geom[crse_lev + 1], geom[crse_lev], 0, q[crse_lev].nComp(), refRatio(crse_lev)); + } + + // compute 1D profile from level 0 multifab + amrex::Box domain = geom[0].Domain(); + auto profile = amrex::sumToLine(q[0], 0, q[0].nComp(), domain, axis); + + // normalize profile + amrex::Long numCells = domain.numPts() / domain.length(axis); + for (double &bin : profile) { + bin /= static_cast(numCells); + } + + return profile; } template void RadhydroSimulation::advanceHydroAtLevelWithRetries(int lev, amrex::Real time, amrex::Real dt_lev, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine) + amrex::YAFluxRegister *fr_as_fine) { - BL_PROFILE_REGION("HydroSolver"); - // timestep retries - const int max_retries = 4; - bool success = false; - - // save the pre-advance fine flux register state in originalFineData - amrex::MultiFab originalFineData; - if (fr_as_fine != nullptr) { - amrex::MultiFab const &fineData = fr_as_fine->getFineData(); - originalFineData.define(fineData.boxArray(), fineData.DistributionMap(), fineData.nComp(), 0); - amrex::Copy(originalFineData, fineData, 0, 0, fineData.nComp(), 0); - } + BL_PROFILE_REGION("HydroSolver"); + // timestep retries + const int max_retries = 4; + bool success = false; + + // save the pre-advance fine flux register state in originalFineData + amrex::MultiFab originalFineData; + if (fr_as_fine != nullptr) { + amrex::MultiFab const &fineData = fr_as_fine->getFineData(); + originalFineData.define(fineData.boxArray(), fineData.DistributionMap(), fineData.nComp(), 0); + amrex::Copy(originalFineData, fineData, 0, 0, fineData.nComp(), 0); + } #ifdef AMREX_PARTICLES - amrex::AmrTracerParticleContainer::ContainerLike originalTracerPC; - if (do_tracers != 0) { - // save the pre-advance tracer particles - originalTracerPC = TracerPC->make_alike(); - } + amrex::AmrTracerParticleContainer::ContainerLike originalTracerPC; + if (do_tracers != 0) { + // save the pre-advance tracer particles + originalTracerPC = TracerPC->make_alike(); + } #endif - for (int retry_count = 0; retry_count <= max_retries; ++retry_count) { - // reduce timestep by a factor of 2^retry_count - const int nsubsteps = std::pow(2, retry_count); - const amrex::Real dt_step = dt_lev / nsubsteps; - - if (retry_count > 0 && Verbose()) { - amrex::Print() << "\t>> Re-trying hydro advance at level " << lev << " with reduced timestep (nsubsteps = " << nsubsteps - << ", dt_new = " << dt_step << ")\n"; - } - - if (retry_count > 0) { - // reset the flux registers to their pre-advance state - if (fr_as_crse != nullptr) { - fr_as_crse->reset(); - } - if (fr_as_fine != nullptr) { - amrex::Copy(fr_as_fine->getFineData(), originalFineData, 0, 0, originalFineData.nComp(), 0); - } + for (int retry_count = 0; retry_count <= max_retries; ++retry_count) { + // reduce timestep by a factor of 2^retry_count + const int nsubsteps = std::pow(2, retry_count); + const amrex::Real dt_step = dt_lev / nsubsteps; + + if (retry_count > 0 && Verbose()) { + amrex::Print() << "\t>> Re-trying hydro advance at level " << lev << " with reduced timestep (nsubsteps = " << nsubsteps + << ", dt_new = " << dt_step << ")\n"; + } + + if (retry_count > 0) { + // reset the flux registers to their pre-advance state + if (fr_as_crse != nullptr) { + fr_as_crse->reset(); + } + if (fr_as_fine != nullptr) { + amrex::Copy(fr_as_fine->getFineData(), originalFineData, 0, 0, originalFineData.nComp(), 0); + } #ifdef AMREX_PARTICLES - if (do_tracers != 0) { - // reset the tracer particles to their pre-advance state - TracerPC->copyParticles(originalTracerPC, true); - } + if (do_tracers != 0) { + // reset the tracer particles to their pre-advance state + TracerPC->copyParticles(originalTracerPC, true); + } #endif - } - - // create temporary multifab for old state - amrex::MultiFab state_old_cc_tmp(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); - amrex::Copy(state_old_cc_tmp, state_old_cc_[lev], 0, 0, Physics_Indices::nvarTotal_cc, nghost_cc_); - - // subcycle advanceHydroAtLevel, checking return value - for (int substep = 0; substep < nsubsteps; ++substep) { - if (substep > 0) { - // since we are starting a new substep, we need to copy hydro state from - // the new state vector to old state vector - amrex::Copy(state_old_cc_tmp, state_new_cc_[lev], 0, 0, ncompHydro_, nghost_cc_); - } - - success = advanceHydroAtLevel(state_old_cc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); - - if (!success) { - if (Verbose()) { - amrex::Print() << "\t>> WARNING: Hydro advance failed on level " << lev << "\n"; - } - break; - } - } - - if (success) { - // we are done, do not attempt more retries - break; - } - } - - if (!success) { - // crash, we have exceeded max_retries - amrex::Print() << "\nQUOKKA FATAL ERROR\n" - << "Hydro update exceeded max_retries on level " << lev << ". Cannot continue, crashing...\n" - << std::endl; - - // write plotfile or Ascent Blueprint file - amrex::ParallelDescriptor::Barrier(); + } + + // create temporary multifab for old state + amrex::MultiFab state_old_cc_tmp(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); + amrex::Copy(state_old_cc_tmp, state_old_cc_[lev], 0, 0, Physics_Indices::nvarTotal_cc, nghost_cc_); + + // subcycle advanceHydroAtLevel, checking return value + for (int substep = 0; substep < nsubsteps; ++substep) { + if (substep > 0) { + // since we are starting a new substep, we need to copy hydro state from + // the new state vector to old state vector + amrex::Copy(state_old_cc_tmp, state_new_cc_[lev], 0, 0, ncompHydro_, nghost_cc_); + } + + success = advanceHydroAtLevel(state_old_cc_tmp, fr_as_crse, fr_as_fine, lev, time, dt_step); + + if (!success) { + if (Verbose()) { + amrex::Print() << "\t>> WARNING: Hydro advance failed on level " << lev << "\n"; + } + break; + } + } + + if (success) { + // we are done, do not attempt more retries + break; + } + } + + if (!success) { + // crash, we have exceeded max_retries + amrex::Print() << "\nQUOKKA FATAL ERROR\n" + << "Hydro update exceeded max_retries on level " << lev << ". Cannot continue, crashing...\n" + << std::endl; + + // write plotfile or Ascent Blueprint file + amrex::ParallelDescriptor::Barrier(); #ifdef AMREX_USE_ASCENT - conduit::Node mesh; - amrex::SingleLevelToBlueprint(state_new_cc_[lev], componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); - conduit::Node bpMeshHost; - bpMeshHost.set(mesh); // copy to host mem (needed for Blueprint HDF5 output) - amrex::WriteBlueprintFiles(bpMeshHost, "debug_hydro_state_fatal", istep[lev] + 1, "hdf5"); + conduit::Node mesh; + amrex::SingleLevelToBlueprint(state_new_cc_[lev], componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); + conduit::Node bpMeshHost; + bpMeshHost.set(mesh); // copy to host mem (needed for Blueprint HDF5 output) + amrex::WriteBlueprintFiles(bpMeshHost, "debug_hydro_state_fatal", istep[lev] + 1, "hdf5"); #else - WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], time, - istep[lev] + 1); + WriteSingleLevelPlotfile(CustomPlotFileName("debug_hydro_state_fatal", istep[lev] + 1), state_new_cc_[lev], componentNames_cc_, geom[lev], time, + istep[lev] + 1); #endif - amrex::ParallelDescriptor::Barrier(); + amrex::ParallelDescriptor::Barrier(); - if (amrex::ParallelDescriptor::IOProcessor()) { - amrex::ParallelDescriptor::Abort(); - } - } + if (amrex::ParallelDescriptor::IOProcessor()) { + amrex::ParallelDescriptor::Abort(); + } + } } template auto RadhydroSimulation::isCflViolated(int lev, amrex::Real time, amrex::Real dt_actual) -> bool { - // check whether dt_actual would violate CFL condition using the post-update hydro state - - // compute max signal speed - amrex::Real max_signal = HydroSystem::maxSignalSpeedLocal(state_new_cc_[lev]); - amrex::ParallelDescriptor::ReduceRealMax(max_signal); - - // compute dt_cfl - auto dx = geom[lev].CellSizeArray(); - const amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); - const amrex::Real dt_cfl = cflNumber_ * (dx_min / max_signal); - - // check whether dt_actual > dt_cfl (CFL violation) - const amrex::Real max_factor = 1.1; - const bool cflViolation = dt_actual > (max_factor * dt_cfl); - if (cflViolation && Verbose()) { - amrex::Print() << "\t>> CFL violation detected on level " << lev << " with dt_lev = " << dt_actual << " and dt_cfl = " << dt_cfl << "\n" - << "\t max_signal = " << max_signal << "\n"; - } - return cflViolation; + // check whether dt_actual would violate CFL condition using the post-update hydro state + + // compute max signal speed + amrex::Real max_signal = HydroSystem::maxSignalSpeedLocal(state_new_cc_[lev]); + amrex::ParallelDescriptor::ReduceRealMax(max_signal); + + // compute dt_cfl + auto dx = geom[lev].CellSizeArray(); + const amrex::Real dx_min = std::min({AMREX_D_DECL(dx[0], dx[1], dx[2])}); + const amrex::Real dt_cfl = cflNumber_ * (dx_min / max_signal); + + // check whether dt_actual > dt_cfl (CFL violation) + const amrex::Real max_factor = 1.1; + const bool cflViolation = dt_actual > (max_factor * dt_cfl); + if (cflViolation && Verbose()) { + amrex::Print() << "\t>> CFL violation detected on level " << lev << " with dt_lev = " << dt_actual << " and dt_cfl = " << dt_cfl << "\n" + << "\t max_signal = " << max_signal << "\n"; + } + return cflViolation; } template void RadhydroSimulation::printCoordinates(int lev, const amrex::IntVect &cell_idx) { - amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); - amrex::Real y_coord = NAN; - amrex::Real z_coord = NAN; + amrex::Real x_coord = geom[lev].ProbLo(0) + (cell_idx[0] + 0.5) * geom[lev].CellSize(0); + amrex::Real y_coord = NAN; + amrex::Real z_coord = NAN; - if (AMREX_SPACEDIM > 1) { - y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); + if (AMREX_SPACEDIM > 1) { + y_coord = geom[lev].ProbLo(1) + (cell_idx[1] + 0.5) * geom[lev].CellSize(1); - if (AMREX_SPACEDIM > 2) { - z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); - } - } - amrex::Print() << "Coordinates: (" << x_coord << ", " << y_coord << ", " << z_coord << "): "; + if (AMREX_SPACEDIM > 2) { + z_coord = geom[lev].ProbLo(2) + (cell_idx[2] + 0.5) * geom[lev].CellSize(2); + } + } + amrex::Print() << "Coordinates: (" << x_coord << ", " << y_coord << ", " << z_coord << "): "; } template auto RadhydroSimulation::advanceHydroAtLevel(amrex::MultiFab &state_old_cc_tmp, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine, - int lev, amrex::Real time, amrex::Real dt_lev) -> bool + int lev, amrex::Real time, amrex::Real dt_lev) -> bool { - BL_PROFILE("RadhydroSimulation::advanceHydroAtLevel()"); - - amrex::Real fluxScaleFactor = NAN; - if (integratorOrder_ == 2) { - fluxScaleFactor = 0.5; - } else if (integratorOrder_ == 1) { - fluxScaleFactor = 1.0; - } - - auto dx = geom[lev].CellSizeArray(); - - // do Strang split source terms (first half-step) - addStrangSplitSourcesWithBuiltin(state_old_cc_tmp, lev, time, 0.5 * dt_lev); - - // create temporary multifab for intermediate state - amrex::MultiFab state_inter_cc_(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); - state_inter_cc_.setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled - - // create temporary multifabs for combined RK2 flux and time-average face velocity - std::array flux_rk2; - std::array avgFaceVel; - const int nghost_vel = 2; // 2 ghost faces are needed for tracer particles - auto ba = grids[lev]; - auto dm = dmap[lev]; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - // initialize flux MultiFab - flux_rk2[idim] = amrex::MultiFab(ba_face, dm, ncompHydro_, 0); - flux_rk2[idim].setVal(0); - // initialize velocity MultiFab - avgFaceVel[idim] = amrex::MultiFab(ba_face, dm, 1, nghost_vel); - avgFaceVel[idim].setVal(0); - } - - // update ghost zones [old timestep] - fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); - - // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) - if (lowLevelDebuggingOutput_ == 1) { + BL_PROFILE("RadhydroSimulation::advanceHydroAtLevel()"); + + amrex::Real fluxScaleFactor = NAN; + if (integratorOrder_ == 2) { + fluxScaleFactor = 0.5; + } else if (integratorOrder_ == 1) { + fluxScaleFactor = 1.0; + } + + auto dx = geom[lev].CellSizeArray(); + + // do Strang split source terms (first half-step) + addStrangSplitSourcesWithBuiltin(state_old_cc_tmp, lev, time, 0.5 * dt_lev); + + // create temporary multifab for intermediate state + amrex::MultiFab state_inter_cc_(grids[lev], dmap[lev], Physics_Indices::nvarTotal_cc, nghost_cc_); + state_inter_cc_.setVal(0); // prevent assert in fillBoundaryConditions when radiation is enabled + + // create temporary multifabs for combined RK2 flux and time-average face velocity + std::array flux_rk2; + std::array avgFaceVel; + const int nghost_vel = 2; // 2 ghost faces are needed for tracer particles + auto ba = grids[lev]; + auto dm = dmap[lev]; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + // initialize flux MultiFab + flux_rk2[idim] = amrex::MultiFab(ba_face, dm, ncompHydro_, 0); + flux_rk2[idim].setVal(0); + // initialize velocity MultiFab + avgFaceVel[idim] = amrex::MultiFab(ba_face, dm, 1, nghost_vel); + avgFaceVel[idim].setVal(0); + } + + // update ghost zones [old timestep] + fillBoundaryConditions(state_old_cc_tmp, state_old_cc_tmp, lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, PostInterpState); + + // LOW LEVEL DEBUGGING: output state_old_cc_tmp (with ghost cells) + if (lowLevelDebuggingOutput_ == 1) { #ifdef AMREX_USE_ASCENT - // write Blueprint HDF5 files - conduit::Node mesh; - amrex::SingleLevelToBlueprint(state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); - amrex::WriteBlueprintFiles(mesh, "debug_stage1_filled_state_old", istep[lev] + 1, "hdf5"); + // write Blueprint HDF5 files + conduit::Node mesh; + amrex::SingleLevelToBlueprint(state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev] + 1, mesh); + amrex::WriteBlueprintFiles(mesh, "debug_stage1_filled_state_old", istep[lev] + 1, "hdf5"); #else - // write AMReX plotfile - // WriteSingleLevelPlotfile(CustomPlotFileName("debug_stage1_filled_state_old", istep[lev]+1), - // state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev]+1); + // write AMReX plotfile + // WriteSingleLevelPlotfile(CustomPlotFileName("debug_stage1_filled_state_old", istep[lev]+1), + // state_old_cc_tmp, componentNames_cc_, geom[lev], time, istep[lev]+1); #endif - } - - // check state validity - AMREX_ASSERT(!state_old_cc_tmp.contains_nan(0, state_old_cc_tmp.nComp())); - AMREX_ASSERT(!state_old_cc_tmp.contains_nan()); // check ghost cells - - auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); - - // Stage 1 of RK2-SSP - { - // advance all grids on local processor (Stage 1 of integrator) - auto const &stateOld = state_old_cc_tmp; - auto &stateNew = state_inter_cc_; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); - amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); - } - - amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); - amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); - redoFlag.setVal(quokka::redoFlag::none); - - HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); - - // LOW LEVEL DEBUGGING: output rhs - if (lowLevelDebuggingOutput_ == 1) { - // write rhs - std::string plotfile_name = CustomPlotFileName("debug_stage1_rhs_fluxes", istep[lev] + 1); - WriteSingleLevelPlotfile(plotfile_name, rhs, componentNames_cc_, geom[lev], time, istep[lev] + 1); - - // write fluxes - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string fullprefix = - amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("Flux_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(fluxArrays[idim], fullprefix); - } - // write face velocities - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string fullprefix = amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", - std::string("FaceVel_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(faceVel[idim], fullprefix); - } - } - - // do first-order flux correction (FOFC) - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long const ncells_bad = redoFlag.sum(0); - if (ncells_bad > 0) { - if (Verbose()) { - amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - // Calculate the coordinates based on the cell index and cell size - printCoordinates(lev, cell_idx); - amrex::print_state(stateNew, cell_idx); - } - - // synchronize redoFlag across ranks - redoFlag.FillBoundary(geom[lev].periodicity()); - - // replace fluxes around troubled cells with Godunov fluxes - replaceFluxes(fluxArrays, FOfluxArrays, redoFlag); - replaceFluxes(faceVel, FOfaceVel, redoFlag); // needed for dual energy - - // re-do RK update - HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); - - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long const ncells_bad = static_cast(redoFlag.sum(0)); - if (ncells_bad > 0) { - // FOFC failed - if (Verbose()) { - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - // print cell state - amrex::Print() << "[FOFC-1] Flux correction failed:\n"; - printCoordinates(lev, cell_idx); - amrex::print_state(stateNew, cell_idx); - amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; - } - if (abortOnFofcFailure_ != 0) { - return false; - } - } - } - - // prevent vacuum - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); - - if (useDualEnergy_ == 1) { - // sync internal energy (requires positive density) - HydroSystem::SyncDualEnergy(stateNew); - } - - if (do_reflux == 1) { - // increment flux registers - incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); - } - } - amrex::Gpu::streamSynchronizeAll(); - - // Stage 2 of RK2-SSP - if (integratorOrder_ == 2) { - // update ghost zones [intermediate stage stored in state_inter_cc_] - fillBoundaryConditions(state_inter_cc_, state_inter_cc_, lev, time + dt_lev, quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // check intermediate state validity - AMREX_ASSERT(!state_inter_cc_.contains_nan(0, state_inter_cc_.nComp())); - AMREX_ASSERT(!state_inter_cc_.contains_nan()); // check ghost zones - - auto const &stateOld = state_old_cc_tmp; - auto const &stateInter = state_inter_cc_; - auto &stateFinal = state_new_cc_[lev]; - auto [fluxArrays, faceVel] = computeHydroFluxes(stateInter, ncompHydro_, lev); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); - amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); - } - - amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); - amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); - redoFlag.setVal(quokka::redoFlag::none); - - HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); - - // do first-order flux correction (FOFC) - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long const ncells_bad = redoFlag.sum(0); - if (ncells_bad > 0) { - if (Verbose()) { - amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - printCoordinates(lev, cell_idx); - amrex::print_state(stateFinal, cell_idx); - } - - // synchronize redoFlag across ranks - redoFlag.FillBoundary(geom[lev].periodicity()); - - // replace fluxes around troubled cells with Godunov fluxes - replaceFluxes(flux_rk2, FOfluxArrays, redoFlag); - replaceFluxes(avgFaceVel, FOfaceVel, redoFlag); // needed for dual energy - - // re-do RK update - HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); - HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); - HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); - - amrex::Gpu::streamSynchronizeAll(); // just in case - amrex::Long ncells_bad = redoFlag.sum(0); - if (ncells_bad > 0) { - // FOFC failed - if (Verbose()) { - const amrex::IntVect cell_idx = redoFlag.maxIndex(0); - // print cell state - amrex::Print() << "[FOFC-2] Flux correction failed:\n"; - printCoordinates(lev, cell_idx); - amrex::print_state(stateFinal, cell_idx); - amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; - } - if (abortOnFofcFailure_ != 0) { - return false; - } - } - } - - // prevent vacuum - HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); - - if (useDualEnergy_ == 1) { - // sync internal energy (requires positive density) - HydroSystem::SyncDualEnergy(stateFinal); - } - - if (do_reflux == 1) { - // increment flux registers - incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); - } - } else { // we are only doing forward Euler - amrex::Copy(state_new_cc_[lev], state_inter_cc_, 0, 0, ncompHydro_, 0); - } - amrex::Gpu::streamSynchronizeAll(); - - // advect tracer particles using avgFaceVel + } + + // check state validity + AMREX_ASSERT(!state_old_cc_tmp.contains_nan(0, state_old_cc_tmp.nComp())); + AMREX_ASSERT(!state_old_cc_tmp.contains_nan()); // check ghost cells + + auto [FOfluxArrays, FOfaceVel] = computeFOHydroFluxes(state_old_cc_tmp, ncompHydro_, lev); + + // Stage 1 of RK2-SSP + { + // advance all grids on local processor (Stage 1 of integrator) + auto const &stateOld = state_old_cc_tmp; + auto &stateNew = state_inter_cc_; + auto [fluxArrays, faceVel] = computeHydroFluxes(stateOld, ncompHydro_, lev); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); + amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); + } + + amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); + amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); + redoFlag.setVal(quokka::redoFlag::none); + + HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); + + // LOW LEVEL DEBUGGING: output rhs + if (lowLevelDebuggingOutput_ == 1) { + // write rhs + std::string plotfile_name = CustomPlotFileName("debug_stage1_rhs_fluxes", istep[lev] + 1); + WriteSingleLevelPlotfile(plotfile_name, rhs, componentNames_cc_, geom[lev], time, istep[lev] + 1); + + // write fluxes + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string fullprefix = + amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("Flux_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(fluxArrays[idim], fullprefix); + } + // write face velocities + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string fullprefix = amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", + std::string("FaceVel_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(faceVel[idim], fullprefix); + } + } + + // do first-order flux correction (FOFC) + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long const ncells_bad = redoFlag.sum(0); + if (ncells_bad > 0) { + if (Verbose()) { + amrex::Print() << "[FOFC-1] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // Calculate the coordinates based on the cell index and cell size + printCoordinates(lev, cell_idx); + amrex::print_state(stateNew, cell_idx); + } + + // synchronize redoFlag across ranks + redoFlag.FillBoundary(geom[lev].periodicity()); + + // replace fluxes around troubled cells with Godunov fluxes + replaceFluxes(fluxArrays, FOfluxArrays, redoFlag); + replaceFluxes(faceVel, FOfaceVel, redoFlag); // needed for dual energy + + // re-do RK update + HydroSystem::ComputeRhsFromFluxes(rhs, fluxArrays, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, faceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateNew, rhs, dt_lev, ncompHydro_, redoFlag); + + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long const ncells_bad = static_cast(redoFlag.sum(0)); + if (ncells_bad > 0) { + // FOFC failed + if (Verbose()) { + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // print cell state + amrex::Print() << "[FOFC-1] Flux correction failed:\n"; + printCoordinates(lev, cell_idx); + amrex::print_state(stateNew, cell_idx); + amrex::Print() << "[FOFC-1] failed for " << ncells_bad << " cells on level " << lev << "\n"; + } + if (abortOnFofcFailure_ != 0) { + return false; + } + } + } + + // prevent vacuum + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateNew); + + if (useDualEnergy_ == 1) { + // sync internal energy (requires positive density) + HydroSystem::SyncDualEnergy(stateNew); + } + + if (do_reflux == 1) { + // increment flux registers + incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); + } + } + amrex::Gpu::streamSynchronizeAll(); + + // Stage 2 of RK2-SSP + if (integratorOrder_ == 2) { + // update ghost zones [intermediate stage stored in state_inter_cc_] + fillBoundaryConditions(state_inter_cc_, state_inter_cc_, lev, time + dt_lev, quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // check intermediate state validity + AMREX_ASSERT(!state_inter_cc_.contains_nan(0, state_inter_cc_.nComp())); + AMREX_ASSERT(!state_inter_cc_.contains_nan()); // check ghost zones + + auto const &stateOld = state_old_cc_tmp; + auto const &stateInter = state_inter_cc_; + auto &stateFinal = state_new_cc_[lev]; + auto [fluxArrays, faceVel] = computeHydroFluxes(stateInter, ncompHydro_, lev); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::MultiFab::Saxpy(flux_rk2[idim], 0.5, fluxArrays[idim], 0, 0, ncompHydro_, 0); + amrex::MultiFab::Saxpy(avgFaceVel[idim], 0.5, faceVel[idim], 0, 0, 1, 0); + } + + amrex::MultiFab rhs(grids[lev], dmap[lev], ncompHydro_, 0); + amrex::iMultiFab redoFlag(grids[lev], dmap[lev], 1, 1); + redoFlag.setVal(quokka::redoFlag::none); + + HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + + // do first-order flux correction (FOFC) + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long const ncells_bad = redoFlag.sum(0); + if (ncells_bad > 0) { + if (Verbose()) { + amrex::Print() << "[FOFC-2] flux correcting " << ncells_bad << " cells on level " << lev << "\n"; + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + printCoordinates(lev, cell_idx); + amrex::print_state(stateFinal, cell_idx); + } + + // synchronize redoFlag across ranks + redoFlag.FillBoundary(geom[lev].periodicity()); + + // replace fluxes around troubled cells with Godunov fluxes + replaceFluxes(flux_rk2, FOfluxArrays, redoFlag); + replaceFluxes(avgFaceVel, FOfaceVel, redoFlag); // needed for dual energy + + // re-do RK update + HydroSystem::ComputeRhsFromFluxes(rhs, flux_rk2, dx, ncompHydro_); + HydroSystem::AddInternalEnergyPdV(rhs, stateOld, dx, avgFaceVel, redoFlag); + HydroSystem::PredictStep(stateOld, stateFinal, rhs, dt_lev, ncompHydro_, redoFlag); + + amrex::Gpu::streamSynchronizeAll(); // just in case + amrex::Long ncells_bad = redoFlag.sum(0); + if (ncells_bad > 0) { + // FOFC failed + if (Verbose()) { + const amrex::IntVect cell_idx = redoFlag.maxIndex(0); + // print cell state + amrex::Print() << "[FOFC-2] Flux correction failed:\n"; + printCoordinates(lev, cell_idx); + amrex::print_state(stateFinal, cell_idx); + amrex::Print() << "[FOFC-2] failed for " << ncells_bad << " cells on level " << lev << "\n"; + } + if (abortOnFofcFailure_ != 0) { + return false; + } + } + } + + // prevent vacuum + HydroSystem::EnforceLimits(densityFloor_, pressureFloor_, speedCeiling_, tempCeiling_, tempFloor_, stateFinal); + + if (useDualEnergy_ == 1) { + // sync internal energy (requires positive density) + HydroSystem::SyncDualEnergy(stateFinal); + } + + if (do_reflux == 1) { + // increment flux registers + incrementFluxRegisters(fr_as_crse, fr_as_fine, fluxArrays, lev, fluxScaleFactor * dt_lev); + } + } else { // we are only doing forward Euler + amrex::Copy(state_new_cc_[lev], state_inter_cc_, 0, 0, ncompHydro_, 0); + } + amrex::Gpu::streamSynchronizeAll(); + + // advect tracer particles using avgFaceVel #ifdef AMREX_PARTICLES - if (do_tracers != 0) { - // copy avgFaceVel to state_new_fc_[lev] - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::Copy(state_new_fc_[lev][idim], avgFaceVel[idim], Physics_Indices::velFirstIndex, 0, - Physics_NumVars::numVelVars_per_dim, nghost_vel); - } - - // fill ghost faces - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - fillBoundaryConditions(state_new_fc_[lev][idim], state_new_fc_[lev][idim], lev, time + 0.5 * dt_lev, quokka::centering::fc, - quokka::direction{idim}, AMRSimulation::InterpHookNone, AMRSimulation::InterpHookNone, - FillPatchType::fillpatch_function); - } - - // copy state_new_fc_[lev] to avgFaceVel - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::Copy(avgFaceVel[idim], state_new_fc_[lev][idim], 0, Physics_Indices::velFirstIndex, - Physics_NumVars::numVelVars_per_dim, nghost_vel); - } - - // advect particles - TracerPC->AdvectWithUmac(avgFaceVel.data(), lev, dt_lev); - } + if (do_tracers != 0) { + // copy avgFaceVel to state_new_fc_[lev] + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::Copy(state_new_fc_[lev][idim], avgFaceVel[idim], Physics_Indices::velFirstIndex, 0, + Physics_NumVars::numVelVars_per_dim, nghost_vel); + } + + // fill ghost faces + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + fillBoundaryConditions(state_new_fc_[lev][idim], state_new_fc_[lev][idim], lev, time + 0.5 * dt_lev, quokka::centering::fc, + quokka::direction{idim}, AMRSimulation::InterpHookNone, AMRSimulation::InterpHookNone, + FillPatchType::fillpatch_function); + } + + // copy state_new_fc_[lev] to avgFaceVel + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + amrex::Copy(avgFaceVel[idim], state_new_fc_[lev][idim], 0, Physics_Indices::velFirstIndex, + Physics_NumVars::numVelVars_per_dim, nghost_vel); + } + + // advect particles + TracerPC->AdvectWithUmac(avgFaceVel.data(), lev, dt_lev); + } #endif - // do Strang split source terms (second half-step) - addStrangSplitSourcesWithBuiltin(state_new_cc_[lev], lev, time + dt_lev, 0.5 * dt_lev); + // do Strang split source terms (second half-step) + addStrangSplitSourcesWithBuiltin(state_new_cc_[lev], lev, time + dt_lev, 0.5 * dt_lev); - // check if we have violated the CFL timestep - return !isCflViolated(lev, time, dt_lev); + // check if we have violated the CFL timestep + return !isCflViolated(lev, time, dt_lev); } template void RadhydroSimulation::replaceFluxes(std::array &fluxes, std::array &FOfluxes, - amrex::iMultiFab &redoFlag) + amrex::iMultiFab &redoFlag) { - BL_PROFILE("RadhydroSimulation::replaceFluxes()"); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // loop over dimension - // ensure that flux arrays have the same number of components - AMREX_ASSERT(fluxes[idim].nComp() == FOfluxes[idim].nComp()); - int ncomp = fluxes[idim].nComp(); - - auto const &FOflux_arrs = FOfluxes[idim].const_arrays(); - auto const &redoFlag_arrs = redoFlag.const_arrays(); - auto flux_arrs = fluxes[idim].arrays(); - - // By convention, the fluxes are defined on the left edge of each zone, - // i.e. flux_(i) is the flux *into* zone i through the interface on the - // left of zone i, and -1.0*flux(i+1) is the flux *into* zone i through - // the interface on the right of zone i. - - amrex::IntVect ng{AMREX_D_DECL(1, 1, 1)}; // must include 1 ghost zone - - amrex::ParallelFor(redoFlag, ng, ncomp, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k, int n) noexcept { - if (redoFlag_arrs[bx](i, j, k) == quokka::redoFlag::redo) { - // replace fluxes with first-order ones at the faces of cell (i,j,k) - if (flux_arrs[bx].contains(i, j, k)) { - flux_arrs[bx](i, j, k, n) = FOflux_arrs[bx](i, j, k, n); - } - if (idim == 0) { // x-dir fluxes - if (flux_arrs[bx].contains(i + 1, j, k)) { - flux_arrs[bx](i + 1, j, k, n) = FOflux_arrs[bx](i + 1, j, k, n); - } - } else if (idim == 1) { // y-dir fluxes - if (flux_arrs[bx].contains(i, j + 1, k)) { - flux_arrs[bx](i, j + 1, k, n) = FOflux_arrs[bx](i, j + 1, k, n); - } - } else if (idim == 2) { // z-dir fluxes - if (flux_arrs[bx].contains(i, j, k + 1)) { - flux_arrs[bx](i, j, k + 1, n) = FOflux_arrs[bx](i, j, k + 1, n); - } - } - } - }); - } + BL_PROFILE("RadhydroSimulation::replaceFluxes()"); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // loop over dimension + // ensure that flux arrays have the same number of components + AMREX_ASSERT(fluxes[idim].nComp() == FOfluxes[idim].nComp()); + int ncomp = fluxes[idim].nComp(); + + auto const &FOflux_arrs = FOfluxes[idim].const_arrays(); + auto const &redoFlag_arrs = redoFlag.const_arrays(); + auto flux_arrs = fluxes[idim].arrays(); + + // By convention, the fluxes are defined on the left edge of each zone, + // i.e. flux_(i) is the flux *into* zone i through the interface on the + // left of zone i, and -1.0*flux(i+1) is the flux *into* zone i through + // the interface on the right of zone i. + + amrex::IntVect ng{AMREX_D_DECL(1, 1, 1)}; // must include 1 ghost zone + + amrex::ParallelFor(redoFlag, ng, ncomp, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k, int n) noexcept { + if (redoFlag_arrs[bx](i, j, k) == quokka::redoFlag::redo) { + // replace fluxes with first-order ones at the faces of cell (i,j,k) + if (flux_arrs[bx].contains(i, j, k)) { + flux_arrs[bx](i, j, k, n) = FOflux_arrs[bx](i, j, k, n); + } + if (idim == 0) { // x-dir fluxes + if (flux_arrs[bx].contains(i + 1, j, k)) { + flux_arrs[bx](i + 1, j, k, n) = FOflux_arrs[bx](i + 1, j, k, n); + } + } else if (idim == 1) { // y-dir fluxes + if (flux_arrs[bx].contains(i, j + 1, k)) { + flux_arrs[bx](i, j + 1, k, n) = FOflux_arrs[bx](i, j + 1, k, n); + } + } else if (idim == 2) { // z-dir fluxes + if (flux_arrs[bx].contains(i, j, k + 1)) { + flux_arrs[bx](i, j, k + 1, n) = FOflux_arrs[bx](i, j, k + 1, n); + } + } + } + }); + } } template void RadhydroSimulation::addFluxArrays(std::array &dstfluxes, - std::array &srcfluxes, const int srccomp, const int dstcomp) + std::array &srcfluxes, const int srccomp, const int dstcomp) { - BL_PROFILE("RadhydroSimulation::addFluxArrays()"); + BL_PROFILE("RadhydroSimulation::addFluxArrays()"); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto const &srcflux = srcfluxes[idim]; - auto &dstflux = dstfluxes[idim]; - amrex::Add(dstflux, srcflux, srccomp, dstcomp, srcflux.nComp(), 0); - } + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto const &srcflux = srcfluxes[idim]; + auto &dstflux = dstfluxes[idim]; + amrex::Add(dstflux, srcflux, srccomp, dstcomp, srcflux.nComp(), 0); + } } template auto RadhydroSimulation::expandFluxArrays(std::array &fluxes, const int nstartNew, const int ncompNew) -> std::array { - BL_PROFILE("RadhydroSimulation::expandFluxArrays()"); - - // This is needed because reflux arrays must have the same number of components as - // state_new_cc_[lev] - auto copyFlux = [nstartNew, ncompNew](amrex::FArrayBox const &oldFlux) { - amrex::Box const &fluxRange = oldFlux.box(); - amrex::FArrayBox newFlux(fluxRange, ncompNew, amrex::The_Async_Arena()); - newFlux.setVal(0.); - // copy oldFlux (starting at 0) to newFlux (starting at nstart) - AMREX_ASSERT(ncompNew >= oldFlux.nComp()); - newFlux.copy(oldFlux, 0, nstartNew, oldFlux.nComp()); - return newFlux; - }; - return {AMREX_D_DECL(copyFlux(fluxes[0]), copyFlux(fluxes[1]), copyFlux(fluxes[2]))}; + BL_PROFILE("RadhydroSimulation::expandFluxArrays()"); + + // This is needed because reflux arrays must have the same number of components as + // state_new_cc_[lev] + auto copyFlux = [nstartNew, ncompNew](amrex::FArrayBox const &oldFlux) { + amrex::Box const &fluxRange = oldFlux.box(); + amrex::FArrayBox newFlux(fluxRange, ncompNew, amrex::The_Async_Arena()); + newFlux.setVal(0.); + // copy oldFlux (starting at 0) to newFlux (starting at nstart) + AMREX_ASSERT(ncompNew >= oldFlux.nComp()); + newFlux.copy(oldFlux, 0, nstartNew, oldFlux.nComp()); + return newFlux; + }; + return {AMREX_D_DECL(copyFlux(fluxes[0]), copyFlux(fluxes[1]), copyFlux(fluxes[2]))}; } template auto RadhydroSimulation::computeHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) -> std::pair, std::array> { - BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); - - auto ba = grids[lev]; - auto dm = dmap[lev]; - const int flatteningGhost = 2; - const int reconstructGhost = 1; - - // allocate temporary MultiFabs - amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); - std::array flatCoefs; - std::array flux; - std::array facevel; - std::array leftState; - std::array rightState; - - for (int idim = 0; idim < 3; ++idim) { - flatCoefs[idim] = amrex::MultiFab(ba, dm, 1, flatteningGhost); - } - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); - rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); - flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); - facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); - } - - // conserved to primitive variables - HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); - - // compute flattening coefficients - AMREX_D_TERM(HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[0], flatteningGhost); - , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[1], flatteningGhost); - , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) - - // compute flux functions - AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars); - , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], - reconstructGhost, nvars);) - - // synchronization point to prevent MultiFabs from going out of scope - amrex::Gpu::streamSynchronizeAll(); - - // LOW LEVEL DEBUGGING: output all of the temporary MultiFabs - if (lowLevelDebuggingOutput_ == 1) { - // write primitive cell-centered state - std::string plotfile_name = CustomPlotFileName("debug_reconstruction", istep[lev] + 1); - WriteSingleLevelPlotfile(plotfile_name, primVar, componentNames_cc_, geom[lev], 0.0, istep[lev] + 1); - - // write flattening coefficients - std::string flatx_filename = CustomPlotFileName("debug_flattening_x", istep[lev] + 1); - std::string flaty_filename = CustomPlotFileName("debug_flattening_y", istep[lev] + 1); - std::string flatz_filename = CustomPlotFileName("debug_flattening_z", istep[lev] + 1); - amrex::Vector flatCompNames{"chi"}; - WriteSingleLevelPlotfile(flatx_filename, flatCoefs[0], flatCompNames, geom[lev], 0.0, istep[lev] + 1); - WriteSingleLevelPlotfile(flaty_filename, flatCoefs[1], flatCompNames, geom[lev], 0.0, istep[lev] + 1); - WriteSingleLevelPlotfile(flatz_filename, flatCoefs[2], flatCompNames, geom[lev], 0.0, istep[lev] + 1); - - // write L interface states - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string const fullprefix = - amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateL_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(leftState[idim], fullprefix); - } - // write R interface states - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (amrex::ParallelDescriptor::IOProcessor()) { - std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); - } - std::string const fullprefix = - amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateR_") + quokka::face_dir_str[idim]); - amrex::VisMF::Write(rightState[idim], fullprefix); - } - } - - // return flux and face-centered velocities - return std::make_pair(std::move(flux), std::move(facevel)); + BL_PROFILE("RadhydroSimulation::computeHydroFluxes()"); + + auto ba = grids[lev]; + auto dm = dmap[lev]; + const int flatteningGhost = 2; + const int reconstructGhost = 1; + + // allocate temporary MultiFabs + amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); + std::array flatCoefs; + std::array flux; + std::array facevel; + std::array leftState; + std::array rightState; + + for (int idim = 0; idim < 3; ++idim) { + flatCoefs[idim] = amrex::MultiFab(ba, dm, 1, flatteningGhost); + } + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); + rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructGhost); + flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); + facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); + } + + // conserved to primitive variables + HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); + + // compute flattening coefficients + AMREX_D_TERM(HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[0], flatteningGhost); + , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[1], flatteningGhost); + , HydroSystem::template ComputeFlatteningCoefficients(primVar, flatCoefs[2], flatteningGhost);) + + // compute flux functions + AMREX_D_TERM(hydroFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], flatCoefs[0], flatCoefs[1], flatCoefs[2], + reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], flatCoefs[0], flatCoefs[1], flatCoefs[2], + reconstructGhost, nvars); + , hydroFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], flatCoefs[0], flatCoefs[1], flatCoefs[2], + reconstructGhost, nvars);) + + // synchronization point to prevent MultiFabs from going out of scope + amrex::Gpu::streamSynchronizeAll(); + + // LOW LEVEL DEBUGGING: output all of the temporary MultiFabs + if (lowLevelDebuggingOutput_ == 1) { + // write primitive cell-centered state + std::string plotfile_name = CustomPlotFileName("debug_reconstruction", istep[lev] + 1); + WriteSingleLevelPlotfile(plotfile_name, primVar, componentNames_cc_, geom[lev], 0.0, istep[lev] + 1); + + // write flattening coefficients + std::string flatx_filename = CustomPlotFileName("debug_flattening_x", istep[lev] + 1); + std::string flaty_filename = CustomPlotFileName("debug_flattening_y", istep[lev] + 1); + std::string flatz_filename = CustomPlotFileName("debug_flattening_z", istep[lev] + 1); + amrex::Vector flatCompNames{"chi"}; + WriteSingleLevelPlotfile(flatx_filename, flatCoefs[0], flatCompNames, geom[lev], 0.0, istep[lev] + 1); + WriteSingleLevelPlotfile(flaty_filename, flatCoefs[1], flatCompNames, geom[lev], 0.0, istep[lev] + 1); + WriteSingleLevelPlotfile(flatz_filename, flatCoefs[2], flatCompNames, geom[lev], 0.0, istep[lev] + 1); + + // write L interface states + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string const fullprefix = + amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateL_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(leftState[idim], fullprefix); + } + // write R interface states + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (amrex::ParallelDescriptor::IOProcessor()) { + std::filesystem::create_directories(plotfile_name + "/raw_fields/Level_" + std::to_string(lev)); + } + std::string const fullprefix = + amrex::MultiFabFileFullPrefix(lev, plotfile_name, "raw_fields/Level_", std::string("StateR_") + quokka::face_dir_str[idim]); + amrex::VisMF::Write(rightState[idim], fullprefix); + } + } + + // return flux and face-centered velocities + return std::make_pair(std::move(flux), std::move(facevel)); } template template void RadhydroSimulation::hydroFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, - amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab const &x1Flat, - amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) + amrex::MultiFab &flux, amrex::MultiFab &faceVel, amrex::MultiFab const &x1Flat, + amrex::MultiFab const &x2Flat, amrex::MultiFab const &x3Flat, const int ng_reconstruct, const int nvars) { - if (reconstructionOrder_ == 3) { - HydroSystem::template ReconstructStatesPPM(primVar, leftState, rightState, ng_reconstruct, nvars); - } else if (reconstructionOrder_ == 2) { - HydroSystem::template ReconstructStatesPLM(primVar, leftState, rightState, ng_reconstruct, nvars); - } else if (reconstructionOrder_ == 1) { - HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); - } else { - amrex::Abort("Invalid reconstruction order specified!"); - } - - // cell-centered kernel - HydroSystem::template FlattenShocks(primVar, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); - - // interface-centered kernel - if constexpr (Physics_Traits::is_mhd_enabled) { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); - } else { - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); - } + if (reconstructionOrder_ == 3) { + HydroSystem::template ReconstructStatesPPM(primVar, leftState, rightState, ng_reconstruct, nvars); + } else if (reconstructionOrder_ == 2) { + HydroSystem::template ReconstructStatesPLM(primVar, leftState, rightState, ng_reconstruct, nvars); + } else if (reconstructionOrder_ == 1) { + HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); + } else { + amrex::Abort("Invalid reconstruction order specified!"); + } + + // cell-centered kernel + HydroSystem::template FlattenShocks(primVar, x1Flat, x2Flat, x3Flat, leftState, rightState, ng_reconstruct, nvars); + + // interface-centered kernel + if constexpr (Physics_Traits::is_mhd_enabled) { + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + } else { + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + } } template auto RadhydroSimulation::computeFOHydroFluxes(amrex::MultiFab const &consVar, const int nvars, const int lev) -> std::pair, std::array> { - BL_PROFILE("RadhydroSimulation::computeFOHydroFluxes()"); - - auto ba = grids[lev]; - auto dm = dmap[lev]; - const int reconstructRange = 1; - - // allocate temporary MultiFabs - amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); - std::array flux; - std::array facevel; - std::array leftState; - std::array rightState; - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); - leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); - rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); - flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); - facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); - } - - // conserved to primitive variables - HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); - - // compute flux functions - AMREX_D_TERM(hydroFOFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], reconstructRange, nvars); - , hydroFOFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], reconstructRange, nvars); - , hydroFOFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], reconstructRange, nvars);) - - // synchronization point to prevent MultiFabs from going out of scope - amrex::Gpu::streamSynchronizeAll(); - - // return flux and face-centered velocities - return std::make_pair(std::move(flux), std::move(facevel)); + BL_PROFILE("RadhydroSimulation::computeFOHydroFluxes()"); + + auto ba = grids[lev]; + auto dm = dmap[lev]; + const int reconstructRange = 1; + + // allocate temporary MultiFabs + amrex::MultiFab primVar(ba, dm, nvars, nghost_cc_); + std::array flux; + std::array facevel; + std::array leftState; + std::array rightState; + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + auto ba_face = amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + leftState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); + rightState[idim] = amrex::MultiFab(ba_face, dm, nvars, reconstructRange); + flux[idim] = amrex::MultiFab(ba_face, dm, nvars, 0); + facevel[idim] = amrex::MultiFab(ba_face, dm, 1, 0); + } + + // conserved to primitive variables + HydroSystem::ConservedToPrimitive(consVar, primVar, nghost_cc_); + + // compute flux functions + AMREX_D_TERM(hydroFOFluxFunction(primVar, leftState[0], rightState[0], flux[0], facevel[0], reconstructRange, nvars); + , hydroFOFluxFunction(primVar, leftState[1], rightState[1], flux[1], facevel[1], reconstructRange, nvars); + , hydroFOFluxFunction(primVar, leftState[2], rightState[2], flux[2], facevel[2], reconstructRange, nvars);) + + // synchronization point to prevent MultiFabs from going out of scope + amrex::Gpu::streamSynchronizeAll(); + + // return flux and face-centered velocities + return std::make_pair(std::move(flux), std::move(facevel)); } template template void RadhydroSimulation::hydroFOFluxFunction(amrex::MultiFab const &primVar, amrex::MultiFab &leftState, amrex::MultiFab &rightState, - amrex::MultiFab &flux, amrex::MultiFab &faceVel, const int ng_reconstruct, const int nvars) + amrex::MultiFab &flux, amrex::MultiFab &faceVel, const int ng_reconstruct, const int nvars) { - // donor-cell reconstruction - HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); - // LLF solver - HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); + // donor-cell reconstruction + HydroSystem::template ReconstructStatesConstant(primVar, leftState, rightState, ng_reconstruct, nvars); + // LLF solver + HydroSystem::template ComputeFluxes(flux, faceVel, leftState, rightState, primVar, artificialViscosityK_); } template void RadhydroSimulation::swapRadiationState(amrex::MultiFab &stateOld, amrex::MultiFab const &stateNew) { - // copy radiation state variables from stateNew to stateOld - amrex::MultiFab::Copy(stateOld, stateNew, nstartHyperbolic_, nstartHyperbolic_, ncompHyperbolic_, 0); + // copy radiation state variables from stateNew to stateOld + amrex::MultiFab::Copy(stateOld, stateNew, nstartHyperbolic_, nstartHyperbolic_, ncompHyperbolic_, 0); } template void RadhydroSimulation::subcycleRadiationAtLevel(int lev, amrex::Real time, amrex::Real dt_lev_hydro, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine) + amrex::YAFluxRegister *fr_as_fine) { - // compute radiation timestep - int nsubSteps = 0; - amrex::Real dt_radiation = NAN; - - if (Physics_Traits::is_hydro_enabled && !(constantDt_ > 0.)) { - // adjust to get integer number of substeps - nsubSteps = computeNumberOfRadiationSubsteps(lev, dt_lev_hydro); - dt_radiation = dt_lev_hydro / static_cast(nsubSteps); - } else { // no hydro, or using constant dt (this is necessary for radiation test problems) - dt_radiation = dt_lev_hydro; - nsubSteps = 1; - } - - if (Verbose() != 0) { - amrex::Print() << "\tRadiation substeps: " << nsubSteps << "\tdt: " << dt_radiation << "\n"; - } - AMREX_ALWAYS_ASSERT(nsubSteps >= 1); - AMREX_ALWAYS_ASSERT(nsubSteps <= (maxSubsteps_ + 1)); - AMREX_ALWAYS_ASSERT(dt_radiation > 0.0); - - // perform subcycle - auto const &dx = geom[lev].CellSizeArray(); - amrex::Real time_subcycle = time; - for (int i = 0; i < nsubSteps; ++i) { - if (i > 0) { - // since we are starting a new substep, we need to copy radiation state from - // new state vector to old state vector - // (this is not necessary for the i=0 substep because we have already swapped - // the full hydro+radiation state vectors at the beginning of the level advance) - swapRadiationState(state_old_cc_[lev], state_new_cc_[lev]); - } - - // We use the IMEX PD-ARS scheme to evolve the radiation subsystem and radiation-matter coupling. - - // Stage 1: advance hyperbolic radiation subsystem using Forward Euler method, starting from state_old_cc_ to state_new_cc_ - advanceRadiationForwardEuler(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); - - // new radiation state is stored in state_new_cc_ - // new hydro state is stored in state_new_cc_ (always the case during radiation update) - - if constexpr (IMEX_a22 > 0.0) { - // matter-radiation exchange source terms of stage 1 - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto const &prob_lo = geom[lev].ProbLoArray(); - auto const &prob_hi = geom[lev].ProbHiArray(); - // update state_new_cc_[lev] in place (updates both radiation and hydro vars) - // Note that only a fraction (IMEX_a32) of the matter-radiation exchange source terms are added to hydro. This ensures that the - // hydro properties get to t + IMEX_a32 dt in terms of matter-radiation exchange. - operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 1, dx, prob_lo, prob_hi); - } - } - - // Stage 2: advance hyperbolic radiation subsystem using midpoint RK2 method, starting from state_old_cc_ to state_new_cc_ - advanceRadiationMidpointRK2(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); - - // new radiation state is stored in state_new_cc_ - // new hydro state is stored in state_new_cc_ (always the case during radiation update) - - // Add the matter-radiation exchange source terms to the radiation subsystem and evolve by (1 - IMEX_a32) * dt - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto const &prob_lo = geom[lev].ProbLoArray(); - auto const &prob_hi = geom[lev].ProbHiArray(); - // update state_new_cc_[lev] in place (updates both radiation and hydro vars) - operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 2, dx, prob_lo, prob_hi); - } - - // new hydro+radiation state is stored in state_new_cc_ - - // update 'time_subcycle' - time_subcycle += dt_radiation; - - // update cell update counter - radiationCellUpdates_ += CountCells(lev); // keep track of number of cell updates - } + // compute radiation timestep + int nsubSteps = 0; + amrex::Real dt_radiation = NAN; + + if (Physics_Traits::is_hydro_enabled && !(constantDt_ > 0.)) { + // adjust to get integer number of substeps + nsubSteps = computeNumberOfRadiationSubsteps(lev, dt_lev_hydro); + dt_radiation = dt_lev_hydro / static_cast(nsubSteps); + } else { // no hydro, or using constant dt (this is necessary for radiation test problems) + dt_radiation = dt_lev_hydro; + nsubSteps = 1; + } + + if (Verbose() != 0) { + amrex::Print() << "\tRadiation substeps: " << nsubSteps << "\tdt: " << dt_radiation << "\n"; + } + AMREX_ALWAYS_ASSERT(nsubSteps >= 1); + AMREX_ALWAYS_ASSERT(nsubSteps <= (maxSubsteps_ + 1)); + AMREX_ALWAYS_ASSERT(dt_radiation > 0.0); + + // perform subcycle + auto const &dx = geom[lev].CellSizeArray(); + amrex::Real time_subcycle = time; + for (int i = 0; i < nsubSteps; ++i) { + if (i > 0) { + // since we are starting a new substep, we need to copy radiation state from + // new state vector to old state vector + // (this is not necessary for the i=0 substep because we have already swapped + // the full hydro+radiation state vectors at the beginning of the level advance) + swapRadiationState(state_old_cc_[lev], state_new_cc_[lev]); + } + + // We use the IMEX PD-ARS scheme to evolve the radiation subsystem and radiation-matter coupling. + + // Stage 1: advance hyperbolic radiation subsystem using Forward Euler method, starting from state_old_cc_ to state_new_cc_ + advanceRadiationForwardEuler(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); + + // new radiation state is stored in state_new_cc_ + // new hydro state is stored in state_new_cc_ (always the case during radiation update) + + if constexpr (IMEX_a22 > 0.0) { + // matter-radiation exchange source terms of stage 1 + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &prob_lo = geom[lev].ProbLoArray(); + auto const &prob_hi = geom[lev].ProbHiArray(); + // update state_new_cc_[lev] in place (updates both radiation and hydro vars) + // Note that only a fraction (IMEX_a32) of the matter-radiation exchange source terms are added to hydro. This ensures that the + // hydro properties get to t + IMEX_a32 dt in terms of matter-radiation exchange. + operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 1, dx, prob_lo, prob_hi); + } + } + + // Stage 2: advance hyperbolic radiation subsystem using midpoint RK2 method, starting from state_old_cc_ to state_new_cc_ + advanceRadiationMidpointRK2(lev, time_subcycle, dt_radiation, i, nsubSteps, fr_as_crse, fr_as_fine); + + // new radiation state is stored in state_new_cc_ + // new hydro state is stored in state_new_cc_ (always the case during radiation update) + + // Add the matter-radiation exchange source terms to the radiation subsystem and evolve by (1 - IMEX_a32) * dt + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto const &prob_lo = geom[lev].ProbLoArray(); + auto const &prob_hi = geom[lev].ProbHiArray(); + // update state_new_cc_[lev] in place (updates both radiation and hydro vars) + operatorSplitSourceTerms(stateNew, indexRange, time_subcycle, dt_radiation, 2, dx, prob_lo, prob_hi); + } + + // new hydro+radiation state is stored in state_new_cc_ + + // update 'time_subcycle' + time_subcycle += dt_radiation; + + // update cell update counter + radiationCellUpdates_ += CountCells(lev); // keep track of number of cell updates + } } template void RadhydroSimulation::advanceRadiationSubstepAtLevel(int lev, amrex::Real time, amrex::Real dt_radiation, int const iter_count, - int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, - amrex::YAFluxRegister *fr_as_fine) + int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, + amrex::YAFluxRegister *fr_as_fine) { - if (Verbose()) { - amrex::Print() << "\tsubstep " << iter_count << " t = " << time << std::endl; - } - - // get cell sizes - auto const &dx = geom[lev].CellSizeArray(); - - // We use the RK2-SSP method here. It needs two registers: one to store the old timestep, - // and another to store the intermediate stage (which is reused for the final stage). - - // update ghost zones [old timestep] - fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 1 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); - - // Stage 1 of RK2-SSP - RadSystem::PredictStep( - stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } - - // update ghost zones [intermediate stage stored in state_new_cc_] - fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 2 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateInter = state_new_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); - - // Stage 2 of RK2-SSP - RadSystem::AddFluxesRK2( - stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } + if (Verbose()) { + amrex::Print() << "\tsubstep " << iter_count << " t = " << time << std::endl; + } + + // get cell sizes + auto const &dx = geom[lev].CellSizeArray(); + + // We use the RK2-SSP method here. It needs two registers: one to store the old timestep, + // and another to store the intermediate stage (which is reused for the final stage). + + // update ghost zones [old timestep] + fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 1 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + + // Stage 1 of RK2-SSP + RadSystem::PredictStep( + stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } + + // update ghost zones [intermediate stage stored in state_new_cc_] + fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 2 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateInter = state_new_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); + + // Stage 2 of RK2-SSP + RadSystem::AddFluxesRK2( + stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } } template void RadhydroSimulation::advanceRadiationForwardEuler(int lev, amrex::Real time, amrex::Real dt_radiation, int const /*iter_count*/, - int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) + int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) { - // get cell sizes - auto const &dx = geom[lev].CellSizeArray(); - - // update ghost zones [old timestep] - fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 1 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); - - // Stage 1 of RK2-SSP - RadSystem::PredictStep( - stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } + // get cell sizes + auto const &dx = geom[lev].CellSizeArray(); + + // update ghost zones [old timestep] + fillBoundaryConditions(state_old_cc_[lev], state_old_cc_[lev], lev, time, quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 1 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + + // Stage 1 of RK2-SSP + RadSystem::PredictStep( + stateOld, stateNew, {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } } template void RadhydroSimulation::advanceRadiationMidpointRK2(int lev, amrex::Real time, amrex::Real dt_radiation, int const /*iter_count*/, - int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) + int const /*nsubsteps*/, amrex::YAFluxRegister *fr_as_crse, amrex::YAFluxRegister *fr_as_fine) { - auto const &dx = geom[lev].CellSizeArray(); - - // update ghost zones [intermediate stage stored in state_new_cc_] - fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, - PostInterpState); - - // advance all grids on local processor (Stage 2 of integrator) - for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &stateOld = state_old_cc_[lev].const_array(iter); - auto const &stateInter = state_new_cc_[lev].const_array(iter); - auto const &stateNew = state_new_cc_[lev].array(iter); - auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); - auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); - - // Stage 2 of RK2-SSP - RadSystem::AddFluxesRK2( - stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, - {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, - {AMREX_D_DECL(fluxDiffusiveArraysOld[0].const_array(), fluxDiffusiveArraysOld[1].const_array(), fluxDiffusiveArraysOld[2].const_array())}, - {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, - dt_radiation, dx, indexRange, ncompHyperbolic_); - - if (do_reflux) { - // increment flux registers - // WARNING: as written, diffusive flux correction is not compatible with reflux!! - auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); - incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); - } - } + auto const &dx = geom[lev].CellSizeArray(); + + // update ghost zones [intermediate stage stored in state_new_cc_] + fillBoundaryConditions(state_new_cc_[lev], state_new_cc_[lev], lev, (time + dt_radiation), quokka::centering::cc, quokka::direction::na, PreInterpState, + PostInterpState); + + // advance all grids on local processor (Stage 2 of integrator) + for (amrex::MFIter iter(state_new_cc_[lev]); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &stateOld = state_old_cc_[lev].const_array(iter); + auto const &stateInter = state_new_cc_[lev].const_array(iter); + auto const &stateNew = state_new_cc_[lev].array(iter); + auto [fluxArraysOld, fluxDiffusiveArraysOld] = computeRadiationFluxes(stateOld, indexRange, ncompHyperbolic_, dx); + auto [fluxArrays, fluxDiffusiveArrays] = computeRadiationFluxes(stateInter, indexRange, ncompHyperbolic_, dx); + + // Stage 2 of RK2-SSP + RadSystem::AddFluxesRK2( + stateNew, stateOld, stateInter, {AMREX_D_DECL(fluxArraysOld[0].array(), fluxArraysOld[1].array(), fluxArraysOld[2].array())}, + {AMREX_D_DECL(fluxArrays[0].array(), fluxArrays[1].array(), fluxArrays[2].array())}, + {AMREX_D_DECL(fluxDiffusiveArraysOld[0].const_array(), fluxDiffusiveArraysOld[1].const_array(), fluxDiffusiveArraysOld[2].const_array())}, + {AMREX_D_DECL(fluxDiffusiveArrays[0].const_array(), fluxDiffusiveArrays[1].const_array(), fluxDiffusiveArrays[2].const_array())}, + dt_radiation, dx, indexRange, ncompHyperbolic_); + + if (do_reflux) { + // increment flux registers + // WARNING: as written, diffusive flux correction is not compatible with reflux!! + auto expandedFluxes = expandFluxArrays(fluxArrays, nstartHyperbolic_, state_new_cc_[lev].nComp()); + incrementFluxRegisters(iter, fr_as_crse, fr_as_fine, expandedFluxes, lev, 0.5 * dt_radiation); + } + } } template void RadhydroSimulation::operatorSplitSourceTerms(amrex::Array4 const &stateNew, const amrex::Box &indexRange, const amrex::Real time, - const double dt, const int stage, amrex::GpuArray const &dx, - amrex::GpuArray const &prob_lo, - amrex::GpuArray const &prob_hi) + const double dt, const int stage, amrex::GpuArray const &dx, + amrex::GpuArray const &prob_lo, + amrex::GpuArray const &prob_hi) { - amrex::FArrayBox radEnergySource(indexRange, Physics_Traits::nGroups, - amrex::The_Async_Arena()); // cell-centered scalar + amrex::FArrayBox radEnergySource(indexRange, Physics_Traits::nGroups, + amrex::The_Async_Arena()); // cell-centered scalar - radEnergySource.setVal(0.); + radEnergySource.setVal(0.); - // cell-centered radiation energy source - RadSystem::SetRadEnergySource(radEnergySource.array(), indexRange, dx, prob_lo, prob_hi, time + dt); + // cell-centered radiation energy source + RadSystem::SetRadEnergySource(radEnergySource.array(), indexRange, dx, prob_lo, prob_hi, time + dt); - // cell-centered source terms - RadSystem::AddSourceTerms(stateNew, radEnergySource.const_array(), indexRange, dt, stage); + // cell-centered source terms + RadSystem::AddSourceTerms(stateNew, radEnergySource.const_array(), indexRange, dt, stage); } template auto RadhydroSimulation::computeRadiationFluxes(amrex::Array4 const &consVar, const amrex::Box &indexRange, const int nvars, - amrex::GpuArray dx) + amrex::GpuArray dx) -> std::tuple, std::array> { - amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, 0); - amrex::FArrayBox x1Flux(x1FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in x - amrex::FArrayBox x1FluxDiffusive(x1FluxRange, nvars, amrex::The_Async_Arena()); + amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, 0); + amrex::FArrayBox x1Flux(x1FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in x + amrex::FArrayBox x1FluxDiffusive(x1FluxRange, nvars, amrex::The_Async_Arena()); #if (AMREX_SPACEDIM >= 2) - amrex::Box const &x2FluxRange = amrex::surroundingNodes(indexRange, 1); - amrex::FArrayBox x2Flux(x2FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in y - amrex::FArrayBox x2FluxDiffusive(x2FluxRange, nvars, amrex::The_Async_Arena()); + amrex::Box const &x2FluxRange = amrex::surroundingNodes(indexRange, 1); + amrex::FArrayBox x2Flux(x2FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in y + amrex::FArrayBox x2FluxDiffusive(x2FluxRange, nvars, amrex::The_Async_Arena()); #endif #if (AMREX_SPACEDIM == 3) - amrex::Box const &x3FluxRange = amrex::surroundingNodes(indexRange, 2); - amrex::FArrayBox x3Flux(x3FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in z - amrex::FArrayBox x3FluxDiffusive(x3FluxRange, nvars, amrex::The_Async_Arena()); + amrex::Box const &x3FluxRange = amrex::surroundingNodes(indexRange, 2); + amrex::FArrayBox x3Flux(x3FluxRange, nvars, amrex::The_Async_Arena()); // node-centered in z + amrex::FArrayBox x3FluxDiffusive(x3FluxRange, nvars, amrex::The_Async_Arena()); #endif - AMREX_D_TERM(fluxFunction(consVar, x1Flux, x1FluxDiffusive, indexRange, nvars, dx); - , fluxFunction(consVar, x2Flux, x2FluxDiffusive, indexRange, nvars, dx); - , fluxFunction(consVar, x3Flux, x3FluxDiffusive, indexRange, nvars, dx);) + AMREX_D_TERM(fluxFunction(consVar, x1Flux, x1FluxDiffusive, indexRange, nvars, dx); + , fluxFunction(consVar, x2Flux, x2FluxDiffusive, indexRange, nvars, dx); + , fluxFunction(consVar, x3Flux, x3FluxDiffusive, indexRange, nvars, dx);) - std::array fluxArrays = {AMREX_D_DECL(std::move(x1Flux), std::move(x2Flux), std::move(x3Flux))}; - std::array fluxDiffusiveArrays{ - AMREX_D_DECL(std::move(x1FluxDiffusive), std::move(x2FluxDiffusive), std::move(x3FluxDiffusive))}; + std::array fluxArrays = {AMREX_D_DECL(std::move(x1Flux), std::move(x2Flux), std::move(x3Flux))}; + std::array fluxDiffusiveArrays{ + AMREX_D_DECL(std::move(x1FluxDiffusive), std::move(x2FluxDiffusive), std::move(x3FluxDiffusive))}; - return std::make_tuple(std::move(fluxArrays), std::move(fluxDiffusiveArrays)); + return std::make_tuple(std::move(fluxArrays), std::move(fluxDiffusiveArrays)); } template template void RadhydroSimulation::fluxFunction(amrex::Array4 const &consState, amrex::FArrayBox &x1Flux, amrex::FArrayBox &x1FluxDiffusive, - const amrex::Box &indexRange, const int nvars, amrex::GpuArray dx) + const amrex::Box &indexRange, const int nvars, amrex::GpuArray dx) { - int dir = 0; - if constexpr (DIR == FluxDir::X1) { - dir = 0; - } else if constexpr (DIR == FluxDir::X2) { - dir = 1; - } else if constexpr (DIR == FluxDir::X3) { - dir = 2; - } - - // extend box to include ghost zones - amrex::Box const &ghostRange = amrex::grow(indexRange, nghost_cc_); - // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to - // work. - amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); - amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, dir); - - amrex::FArrayBox primVar(ghostRange, nvars, amrex::The_Async_Arena()); - amrex::FArrayBox x1LeftState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); - amrex::FArrayBox x1RightState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); - - // cell-centered kernel - RadSystem::ConservedToPrimitive(consState, primVar.array(), ghostRange); - - if (radiationReconstructionOrder_ == 3) { - // mixed interface/cell-centered kernel - RadSystem::template ReconstructStatesPPM(primVar.array(), x1LeftState.array(), x1RightState.array(), reconstructRange, - x1ReconstructRange, nvars); - } else if (radiationReconstructionOrder_ == 2) { - // PLM and donor cell are interface-centered kernels - RadSystem::template ReconstructStatesPLM(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, nvars); - } else if (radiationReconstructionOrder_ == 1) { - RadSystem::template ReconstructStatesConstant(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, - nvars); - } else { - amrex::Abort("Invalid reconstruction order for radiation variables! Aborting..."); - } - - // interface-centered kernel - amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, dir); - RadSystem::template ComputeFluxes(x1Flux.array(), x1FluxDiffusive.array(), x1LeftState.array(), x1RightState.array(), x1FluxRange, - consState, - dx); // watch out for argument order!! + int dir = 0; + if constexpr (DIR == FluxDir::X1) { + dir = 0; + } else if constexpr (DIR == FluxDir::X2) { + dir = 1; + } else if constexpr (DIR == FluxDir::X3) { + dir = 2; + } + + // extend box to include ghost zones + amrex::Box const &ghostRange = amrex::grow(indexRange, nghost_cc_); + // N.B.: A one-zone layer around the cells must be fully reconstructed in order for PPM to + // work. + amrex::Box const &reconstructRange = amrex::grow(indexRange, 1); + amrex::Box const &x1ReconstructRange = amrex::surroundingNodes(reconstructRange, dir); + + amrex::FArrayBox primVar(ghostRange, nvars, amrex::The_Async_Arena()); + amrex::FArrayBox x1LeftState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); + amrex::FArrayBox x1RightState(x1ReconstructRange, nvars, amrex::The_Async_Arena()); + + // cell-centered kernel + RadSystem::ConservedToPrimitive(consState, primVar.array(), ghostRange); + + if (radiationReconstructionOrder_ == 3) { + // mixed interface/cell-centered kernel + RadSystem::template ReconstructStatesPPM(primVar.array(), x1LeftState.array(), x1RightState.array(), reconstructRange, + x1ReconstructRange, nvars); + } else if (radiationReconstructionOrder_ == 2) { + // PLM and donor cell are interface-centered kernels + RadSystem::template ReconstructStatesPLM(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, nvars); + } else if (radiationReconstructionOrder_ == 1) { + RadSystem::template ReconstructStatesConstant(primVar.array(), x1LeftState.array(), x1RightState.array(), x1ReconstructRange, + nvars); + } else { + amrex::Abort("Invalid reconstruction order for radiation variables! Aborting..."); + } + + // interface-centered kernel + amrex::Box const &x1FluxRange = amrex::surroundingNodes(indexRange, dir); + RadSystem::template ComputeFluxes(x1Flux.array(), x1FluxDiffusive.array(), x1LeftState.array(), x1RightState.array(), x1FluxRange, + consState, + dx); // watch out for argument order!! } #endif // RADIATION_SIMULATION_HPP_ From 998694b94abbdd0da02a3d068c63f0e62b6c6a5e Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 14:38:12 +0100 Subject: [PATCH 128/165] debug chemistry --- src/Chemistry.hpp | 212 ++++++++++++++++++++++++---------------------- 1 file changed, 113 insertions(+), 99 deletions(-) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index 1a30a0c99..6775bb549 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -31,107 +31,121 @@ namespace quokka::chemistry AMREX_GPU_DEVICE void chemburner(burn_t &chemstate, const Real dt); -template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed) +template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed, const Real min_density_allowed) { - for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &state = mf.array(iter); - - amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const Real rho = state(i, j, k, HydroSystem::density_index); - const Real xmom = state(i, j, k, HydroSystem::x1Momentum_index); - const Real ymom = state(i, j, k, HydroSystem::x2Momentum_index); - const Real zmom = state(i, j, k, HydroSystem::x3Momentum_index); - const Real Ener = state(i, j, k, HydroSystem::energy_index); - const Real Eint = RadSystem::ComputeEintFromEgas(rho, xmom, ymom, zmom, Ener); - - std::array chem = {-1.0}; - std::array inmfracs = {-1.0}; - Real insum = 0.0_rt; - - for (int nn = 0; nn < NumSpec; ++nn) { - chem[nn] = state(i, j, k, HydroSystem::scalar0_index + nn) / - rho; // state has partial densities, so divide by rho to get mass fractions - } - - // do chemistry using microphysics - - burn_t chemstate; - - for (int nn = 0; nn < NumSpec; ++nn) { - inmfracs[nn] = chem[nn] * rho / spmasses[nn]; - chemstate.xn[nn] = inmfracs[nn]; - } - - // stop the test if we have reached very high densities - if (rho > max_density_allowed) { - amrex::Abort("Density exceeded max_density_allowed!"); - } - - // input density and eint in burn state - // Microphysics needs specific eint - chemstate.rho = rho; - chemstate.e = Eint / rho; - - // call the EOS to set initial internal energy e - eos(eos_input_re, chemstate); - - // do the actual integration - // do it in .cpp so that it is not built at compile time for all tests - // which would otherwise slow down compilation due to the large RHS file - chemburner(chemstate, dt); - - if (!chemstate.success) { - amrex::Abort("VODE integration was unsuccessful!"); - } - - // ensure positivity and normalize - for (int nn = 0; nn < NumSpec; ++nn) { - chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); - inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; - insum += inmfracs[nn]; - } - - for (int nn = 0; nn < NumSpec; ++nn) { - inmfracs[nn] /= insum; - // update the number densities with conserved mass fractions - chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; - } - - // update the number density of electrons due to charge conservation - // TODO(psharda): generalize this to other chem networks - chemstate.xn[0] = -chemstate.xn[3] - chemstate.xn[7] + chemstate.xn[1] + chemstate.xn[12] + chemstate.xn[6] + chemstate.xn[4] + - chemstate.xn[9] + 2.0 * chemstate.xn[11]; - - // reconserve mass fractions post charge conservation - insum = 0; - for (int nn = 0; nn < NumSpec; ++nn) { - chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); - inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; - insum += inmfracs[nn]; - } - - for (int nn = 0; nn < NumSpec; ++nn) { - inmfracs[nn] /= insum; - // update the number densities with conserved mass fractions - chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; - } - - // get the updated specific eint - eos(eos_input_rt, chemstate); - - // get dEint - // Quokka uses rho*eint - const Real dEint = (chemstate.e * chemstate.rho) - Eint; - state(i, j, k, HydroSystem::internalEnergy_index) += dEint; - state(i, j, k, HydroSystem::energy_index) += dEint; - - for (int nn = 0; nn < NumSpec; ++nn) { - state(i, j, k, HydroSystem::scalar0_index + nn) = inmfracs[nn] * rho; // scale by rho to return partial densities - } - }); - } + BL_PROFILE("Chemistry::computeChemistry()"); + for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &state = mf.array(iter); + + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const Real rho = state(i, j, k, HydroSystem::density_index); + const Real xmom = state(i, j, k, HydroSystem::x1Momentum_index); + const Real ymom = state(i, j, k, HydroSystem::x2Momentum_index); + const Real zmom = state(i, j, k, HydroSystem::x3Momentum_index); + const Real Ener = state(i, j, k, HydroSystem::energy_index); + const Real Eint = RadSystem::ComputeEintFromEgas(rho, xmom, ymom, zmom, Ener); + + std::array chem = {-1.0}; + std::array inmfracs = {-1.0}; + Real insum = 0.0_rt; + + for (int nn = 0; nn < NumSpec; ++nn) { + chem[nn] = state(i, j, k, HydroSystem::scalar0_index + nn) / + rho; // state has partial densities, so divide by rho to get mass fractions + } + + // do chemistry using microphysics + + burn_t chemstate; + + for (int nn = 0; nn < NumSpec; ++nn) { + inmfracs[nn] = chem[nn] * rho / spmasses[nn]; + chemstate.xn[nn] = inmfracs[nn]; + } + + // dont do chemistry in cells with densities below the minimum density specified + if (rho < min_density_allowed) { + return; + } + + // stop the test if we have reached very high densities + if (rho > max_density_allowed) { + amrex::Abort("Density exceeded max_density_allowed!"); + } + + // input density and eint in burn state + // Microphysics needs specific eint + chemstate.rho = rho; + chemstate.e = Eint / rho; + + // call the EOS to set initial internal energy e + eos(eos_input_re, chemstate); + + if (dt < 0) { + amrex::Abort("Cannot do chemistry with dt < 0!"); + } + + // do the actual integration + // do it in .cpp so that it is not built at compile time for all tests + // which would otherwise slow down compilation due to the large RHS file + chemburner(chemstate, dt); + + if (std::isnan(chemstate.xn[0]) || std::isnan(chemstate.rho)) { + amrex::Abort("Burner returned NAN"); + } + + if (!chemstate.success) { + amrex::Abort("VODE integration was unsuccessful!"); + } + + // ensure positivity and normalize + for (int nn = 0; nn < NumSpec; ++nn) { + chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); + inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; + insum += inmfracs[nn]; + } + + for (int nn = 0; nn < NumSpec; ++nn) { + inmfracs[nn] /= insum; + // update the number densities with conserved mass fractions + chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; + } + + // update the number density of electrons due to charge conservation + // TODO(psharda): generalize this to other chem networks + chemstate.xn[0] = -chemstate.xn[3] - chemstate.xn[7] + chemstate.xn[1] + chemstate.xn[12] + chemstate.xn[6] + chemstate.xn[4] + + chemstate.xn[9] + 2.0 * chemstate.xn[11]; + + // reconserve mass fractions post charge conservation + insum = 0; + for (int nn = 0; nn < NumSpec; ++nn) { + chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); + inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; + insum += inmfracs[nn]; + } + + for (int nn = 0; nn < NumSpec; ++nn) { + inmfracs[nn] /= insum; + // update the number densities with conserved mass fractions + chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; + } + + // get the updated specific eint + eos(eos_input_rt, chemstate); + + // get dEint + // Quokka uses rho*eint + const Real dEint = (chemstate.e * chemstate.rho) - Eint; + state(i, j, k, HydroSystem::internalEnergy_index) += dEint; + state(i, j, k, HydroSystem::energy_index) += dEint; + + for (int nn = 0; nn < NumSpec; ++nn) { + state(i, j, k, HydroSystem::scalar0_index + nn) = inmfracs[nn] * rho; // scale by rho to return partial densities + } + }); + } } } // namespace quokka::chemistry From 021c7a2a584f6b218271fce57061087527f0471b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:38:28 +0000 Subject: [PATCH 129/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/Chemistry.hpp | 224 +++++++++++++++++++++++----------------------- 1 file changed, 112 insertions(+), 112 deletions(-) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index 6775bb549..ed2e79997 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -34,118 +34,118 @@ AMREX_GPU_DEVICE void chemburner(burn_t &chemstate, const Real dt); template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed, const Real min_density_allowed) { - BL_PROFILE("Chemistry::computeChemistry()"); - for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { - const amrex::Box &indexRange = iter.validbox(); - auto const &state = mf.array(iter); - - amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const Real rho = state(i, j, k, HydroSystem::density_index); - const Real xmom = state(i, j, k, HydroSystem::x1Momentum_index); - const Real ymom = state(i, j, k, HydroSystem::x2Momentum_index); - const Real zmom = state(i, j, k, HydroSystem::x3Momentum_index); - const Real Ener = state(i, j, k, HydroSystem::energy_index); - const Real Eint = RadSystem::ComputeEintFromEgas(rho, xmom, ymom, zmom, Ener); - - std::array chem = {-1.0}; - std::array inmfracs = {-1.0}; - Real insum = 0.0_rt; - - for (int nn = 0; nn < NumSpec; ++nn) { - chem[nn] = state(i, j, k, HydroSystem::scalar0_index + nn) / - rho; // state has partial densities, so divide by rho to get mass fractions - } - - // do chemistry using microphysics - - burn_t chemstate; - - for (int nn = 0; nn < NumSpec; ++nn) { - inmfracs[nn] = chem[nn] * rho / spmasses[nn]; - chemstate.xn[nn] = inmfracs[nn]; - } - - // dont do chemistry in cells with densities below the minimum density specified - if (rho < min_density_allowed) { - return; - } - - // stop the test if we have reached very high densities - if (rho > max_density_allowed) { - amrex::Abort("Density exceeded max_density_allowed!"); - } - - // input density and eint in burn state - // Microphysics needs specific eint - chemstate.rho = rho; - chemstate.e = Eint / rho; - - // call the EOS to set initial internal energy e - eos(eos_input_re, chemstate); - - if (dt < 0) { - amrex::Abort("Cannot do chemistry with dt < 0!"); - } - - // do the actual integration - // do it in .cpp so that it is not built at compile time for all tests - // which would otherwise slow down compilation due to the large RHS file - chemburner(chemstate, dt); - - if (std::isnan(chemstate.xn[0]) || std::isnan(chemstate.rho)) { - amrex::Abort("Burner returned NAN"); - } - - if (!chemstate.success) { - amrex::Abort("VODE integration was unsuccessful!"); - } - - // ensure positivity and normalize - for (int nn = 0; nn < NumSpec; ++nn) { - chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); - inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; - insum += inmfracs[nn]; - } - - for (int nn = 0; nn < NumSpec; ++nn) { - inmfracs[nn] /= insum; - // update the number densities with conserved mass fractions - chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; - } - - // update the number density of electrons due to charge conservation - // TODO(psharda): generalize this to other chem networks - chemstate.xn[0] = -chemstate.xn[3] - chemstate.xn[7] + chemstate.xn[1] + chemstate.xn[12] + chemstate.xn[6] + chemstate.xn[4] + - chemstate.xn[9] + 2.0 * chemstate.xn[11]; - - // reconserve mass fractions post charge conservation - insum = 0; - for (int nn = 0; nn < NumSpec; ++nn) { - chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); - inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; - insum += inmfracs[nn]; - } - - for (int nn = 0; nn < NumSpec; ++nn) { - inmfracs[nn] /= insum; - // update the number densities with conserved mass fractions - chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; - } - - // get the updated specific eint - eos(eos_input_rt, chemstate); - - // get dEint - // Quokka uses rho*eint - const Real dEint = (chemstate.e * chemstate.rho) - Eint; - state(i, j, k, HydroSystem::internalEnergy_index) += dEint; - state(i, j, k, HydroSystem::energy_index) += dEint; - - for (int nn = 0; nn < NumSpec; ++nn) { - state(i, j, k, HydroSystem::scalar0_index + nn) = inmfracs[nn] * rho; // scale by rho to return partial densities - } - }); - } + BL_PROFILE("Chemistry::computeChemistry()"); + for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { + const amrex::Box &indexRange = iter.validbox(); + auto const &state = mf.array(iter); + + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const Real rho = state(i, j, k, HydroSystem::density_index); + const Real xmom = state(i, j, k, HydroSystem::x1Momentum_index); + const Real ymom = state(i, j, k, HydroSystem::x2Momentum_index); + const Real zmom = state(i, j, k, HydroSystem::x3Momentum_index); + const Real Ener = state(i, j, k, HydroSystem::energy_index); + const Real Eint = RadSystem::ComputeEintFromEgas(rho, xmom, ymom, zmom, Ener); + + std::array chem = {-1.0}; + std::array inmfracs = {-1.0}; + Real insum = 0.0_rt; + + for (int nn = 0; nn < NumSpec; ++nn) { + chem[nn] = state(i, j, k, HydroSystem::scalar0_index + nn) / + rho; // state has partial densities, so divide by rho to get mass fractions + } + + // do chemistry using microphysics + + burn_t chemstate; + + for (int nn = 0; nn < NumSpec; ++nn) { + inmfracs[nn] = chem[nn] * rho / spmasses[nn]; + chemstate.xn[nn] = inmfracs[nn]; + } + + // dont do chemistry in cells with densities below the minimum density specified + if (rho < min_density_allowed) { + return; + } + + // stop the test if we have reached very high densities + if (rho > max_density_allowed) { + amrex::Abort("Density exceeded max_density_allowed!"); + } + + // input density and eint in burn state + // Microphysics needs specific eint + chemstate.rho = rho; + chemstate.e = Eint / rho; + + // call the EOS to set initial internal energy e + eos(eos_input_re, chemstate); + + if (dt < 0) { + amrex::Abort("Cannot do chemistry with dt < 0!"); + } + + // do the actual integration + // do it in .cpp so that it is not built at compile time for all tests + // which would otherwise slow down compilation due to the large RHS file + chemburner(chemstate, dt); + + if (std::isnan(chemstate.xn[0]) || std::isnan(chemstate.rho)) { + amrex::Abort("Burner returned NAN"); + } + + if (!chemstate.success) { + amrex::Abort("VODE integration was unsuccessful!"); + } + + // ensure positivity and normalize + for (int nn = 0; nn < NumSpec; ++nn) { + chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); + inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; + insum += inmfracs[nn]; + } + + for (int nn = 0; nn < NumSpec; ++nn) { + inmfracs[nn] /= insum; + // update the number densities with conserved mass fractions + chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; + } + + // update the number density of electrons due to charge conservation + // TODO(psharda): generalize this to other chem networks + chemstate.xn[0] = -chemstate.xn[3] - chemstate.xn[7] + chemstate.xn[1] + chemstate.xn[12] + chemstate.xn[6] + chemstate.xn[4] + + chemstate.xn[9] + 2.0 * chemstate.xn[11]; + + // reconserve mass fractions post charge conservation + insum = 0; + for (int nn = 0; nn < NumSpec; ++nn) { + chemstate.xn[nn] = amrex::max(chemstate.xn[nn], small_x); + inmfracs[nn] = spmasses[nn] * chemstate.xn[nn] / chemstate.rho; + insum += inmfracs[nn]; + } + + for (int nn = 0; nn < NumSpec; ++nn) { + inmfracs[nn] /= insum; + // update the number densities with conserved mass fractions + chemstate.xn[nn] = inmfracs[nn] * chemstate.rho / spmasses[nn]; + } + + // get the updated specific eint + eos(eos_input_rt, chemstate); + + // get dEint + // Quokka uses rho*eint + const Real dEint = (chemstate.e * chemstate.rho) - Eint; + state(i, j, k, HydroSystem::internalEnergy_index) += dEint; + state(i, j, k, HydroSystem::energy_index) += dEint; + + for (int nn = 0; nn < NumSpec; ++nn) { + state(i, j, k, HydroSystem::scalar0_index + nn) = inmfracs[nn] * rho; // scale by rho to return partial densities + } + }); + } } } // namespace quokka::chemistry From 1cac4d44dd0ebcd359d72d05c9c388ee6ebcb5fc Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 19:10:25 +0100 Subject: [PATCH 130/165] const tiny profiler --- src/Chemistry.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index ed2e79997..da8dfdf52 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -34,7 +34,7 @@ AMREX_GPU_DEVICE void chemburner(burn_t &chemstate, const Real dt); template void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed, const Real min_density_allowed) { - BL_PROFILE("Chemistry::computeChemistry()"); + const BL_PROFILE("Chemistry::computeChemistry()"); for (amrex::MFIter iter(mf); iter.isValid(); ++iter) { const amrex::Box &indexRange = iter.validbox(); auto const &state = mf.array(iter); From a8a490c503329d193d1ddbafe81d8dc2a23ffec2 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 19:25:08 +0100 Subject: [PATCH 131/165] moved turb files to src --- src/TurbDataReader.cpp | 119 ++++++++++++++++ src/TurbDataReader.hpp | 59 ++++++++ src/perturbation.py | 311 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 489 insertions(+) create mode 100644 src/TurbDataReader.cpp create mode 100644 src/TurbDataReader.hpp create mode 100644 src/perturbation.py diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp new file mode 100644 index 000000000..6834c17a1 --- /dev/null +++ b/src/TurbDataReader.cpp @@ -0,0 +1,119 @@ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file TurbDataReader.cpp +/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. +/// + +#include "TurbDataReader.hpp" +#include "AMReX_Arena.H" +#include "AMReX_BLassert.H" +#include "AMReX_Print.H" +#include "AMReX_TableData.H" +#include + +auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D +{ + // open dataset + hid_t dset_id = 0; + dset_id = H5Dopen2(file_id, dataset_name, H5P_DEFAULT); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dset_id != -1, "Can't open table!"); + + // get dimensions + hid_t const dspace = H5Dget_space(dset_id); + const int ndims = H5Sget_simple_extent_ndims(dspace); + std::vector dims(ndims); + H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); + + uint64_t data_size = 1; + for (int idim = 0; idim < ndims; ++idim) { + data_size *= dims[idim]; + } + + // allocate array for dataset storage + auto *temp_data = new double[data_size]; + + // read dataset + herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); + + // close dataset + status = H5Dclose(dset_id); + + // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 + // tables use row-major (C-order) indexing! + amrex::GpuArray const lo{0, 0, 0}; + amrex::GpuArray const hi{static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])}; + auto table = amrex::Table3D(temp_data, lo, hi); + return table; +} + +void initialize_turbdata(turb_data &data, std::string &data_file) +{ + amrex::Print() << "Initializing turbulence data...\n"; + amrex::Print() << fmt::format("data_file: {}.\n", data_file); + + herr_t status = 0; + herr_t const h5_error = -1; + + // open file + hid_t file_id = 0; + file_id = H5Fopen(data_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(file_id != h5_error, "Failed to open data file!"); + + data.dvx = read_dataset(file_id, "pertx"); + data.dvy = read_dataset(file_id, "perty"); + data.dvz = read_dataset(file_id, "pertz"); + + // close file + status = H5Fclose(file_id); +} + +auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData +{ + amrex::Array tlo{in_t.begin[0], in_t.begin[1], in_t.begin[2]}; + amrex::Array thi{in_t.end[0] - 1, in_t.end[1] - 1, in_t.end[2] - 1}; + amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); + auto h_table = tableData.table(); + + amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; + + // fill tableData + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + for (int k = tlo[2]; k <= thi[2]; ++k) { + h_table(i, j, k) = in_t(i, j, k); + } + } + } + + return tableData; +} + +auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real +{ + amrex::Array tlo = dvx.lo(); + amrex::Array thi = dvx.hi(); + auto const &dvx_table = dvx.const_table(); + auto const &dvy_table = dvy.const_table(); + auto const &dvz_table = dvz.const_table(); + + // compute rms power + amrex::Real rms_sq = 0; + amrex::Long N = 0; + for (int i = tlo[0]; i <= thi[0]; ++i) { + for (int j = tlo[1]; j <= thi[1]; ++j) { + for (int k = tlo[2]; k <= thi[2]; ++k) { + amrex::Real const vx = dvx_table(i, j, k); + amrex::Real const vy = dvy_table(i, j, k); + amrex::Real const vz = dvz_table(i, j, k); + rms_sq += vx * vx + vy * vy + vz * vz; + ++N; + } + } + } + rms_sq /= static_cast(N); + return std::sqrt(rms_sq); +} diff --git a/src/TurbDataReader.hpp b/src/TurbDataReader.hpp new file mode 100644 index 000000000..b127cc92a --- /dev/null +++ b/src/TurbDataReader.hpp @@ -0,0 +1,59 @@ +#ifndef TURBDATAREADER_HPP_ // NOLINT +#define TURBDATAREADER_HPP_ +//============================================================================== +// TwoMomentRad - a radiation transport library for patch-based AMR codes +// Copyright 2020 Benjamin Wibking. +// Released under the MIT license. See LICENSE file included in the GitHub repo. +//============================================================================== +/// \file TurbDataReader.hpp +/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. +/// + +#include +#include +#include +#include +#include + +#include "fmt/core.h" +#include +#include +#include + +#include "AMReX.H" +#include "AMReX_Array.H" +#include "AMReX_BLassert.H" +#include "AMReX_TableData.H" + +/* HDF5 definitions */ + +#define HDF5_FILE_I4 H5T_STD_I32BE +#define HDF5_FILE_I8 H5T_STD_I64BE +#define HDF5_FILE_R4 H5T_IEEE_F32BE +#define HDF5_FILE_R8 H5T_IEEE_F64BE +#define HDF5_FILE_B8 H5T_STD_B8BE + +#define HDF5_I4 H5T_NATIVE_INT +#define HDF5_I8 H5T_NATIVE_LLONG +#define HDF5_R4 H5T_NATIVE_FLOAT +#define HDF5_R8 H5T_NATIVE_DOUBLE +#define HDF5_R16 H5T_NATIVE_LDOUBLE + +// Cooling table storage + +using turb_data = struct turb_data { + // values + amrex::Table3D dvx; + amrex::Table3D dvy; + amrex::Table3D dvz; +}; + +void initialize_turbdata(turb_data &data, std::string &data_file); + +auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D; + +auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData; + +auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real; + +#endif // TURBDATAREADER_HPP_ diff --git a/src/perturbation.py b/src/perturbation.py new file mode 100644 index 000000000..888d1756c --- /dev/null +++ b/src/perturbation.py @@ -0,0 +1,311 @@ +import h5py +import numpy as np +import numpy.fft +from math import * +from optparse import OptionParser +import sys + + +def init_perturbations(n, kmin, kmax, dtype): + kx = np.zeros(n, dtype=dtype) + ky = np.zeros(n, dtype=dtype) + kz = np.zeros(n, dtype=dtype) + # perform fft k-ordering convention shifts + for j in range(0,n[1]): + for k in range(0,n[2]): + kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) + for i in range(0,n[0]): + for k in range(0,n[2]): + ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) + for i in range(0,n[0]): + for j in range(0,n[1]): + kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) + + kx = np.array(kx, dtype=dtype) + ky = np.array(ky, dtype=dtype) + kz = np.array(kz, dtype=dtype) + k = np.sqrt(np.array(kx**2+ky**2+kz**2, dtype=dtype)) + + # only use the positive frequencies + inds = np.where(np.logical_and(k**2 >= kmin**2, k**2 < (kmax+1)**2)) + nr = len(inds[0]) + + phasex = np.zeros(n, dtype=dtype) + phasex[inds] = 2.*pi*np.random.uniform(size=nr) + fx = np.zeros(n, dtype=dtype) + fx[inds] = np.random.normal(size=nr) + + phasey = np.zeros(n, dtype=dtype) + phasey[inds] = 2.*pi*np.random.uniform(size=nr) + fy = np.zeros(n, dtype=dtype) + fy[inds] = np.random.normal(size=nr) + + phasez = np.zeros(n, dtype=dtype) + phasez[inds] = 2.*pi*np.random.uniform(size=nr) + fz = np.zeros(n, dtype=dtype) + fz[inds] = np.random.normal(size=nr) + + # rescale perturbation amplitude so that low number statistics + # at low k do not throw off the desired power law scaling. + for i in range(kmin, kmax+1): + slice_inds = np.where(np.logical_and(k >= i, k < i+1)) + rescale = sqrt(np.sum(np.abs(fx[slice_inds])**2 + np.abs(fy[slice_inds])**2 + np.abs(fz[slice_inds])**2)) + fx[slice_inds] = fx[slice_inds]/rescale + fy[slice_inds] = fy[slice_inds]/rescale + fz[slice_inds] = fz[slice_inds]/rescale + + # set the power law behavior + # wave number bins + fx[inds] = fx[inds]*k[inds]**-(0.5*alpha) + fy[inds] = fy[inds]*k[inds]**-(0.5*alpha) + fz[inds] = fz[inds]*k[inds]**-(0.5*alpha) + + # add in phases + fx = np.cos(phasex)*fx + 1j*np.sin(phasex)*fx + fy = np.cos(phasey)*fy + 1j*np.sin(phasey)*fy + fz = np.cos(phasez)*fz + 1j*np.sin(phasez)*fz + + return fx, fy, fz, kx, ky, kz + + +def normalize(fx, fy, fz): + norm = np.sqrt(np.sum(fx**2 + fy**2 + fz**2)/np.product(n)) + fx = fx/norm + fy = fy/norm + fz = fz/norm + return fx, fy, fz + + +def make_perturbations(n, kmin, kmax, f_solenoidal): + fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) + if f_solenoidal != None: + k2 = kx**2+ky**2+kz**2 + # solenoidal part + fxs = 0.; fys =0.; fzs = 0. + if f_solenoidal != 0.0: + fxs = np.real(fx - kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fys = np.real(fy - ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fzs = np.real(fz - kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + ind = np.where(k2 == 0) + fxs[ind] = 0.; fys[ind] = 0.; fzs[ind] = 0. + # need to normalize this before applying relative weighting of solenoidal / compressive components + norm = np.sqrt(np.sum(fxs**2+fys**2+fzs**2)) + fxs = fxs/norm + fys = fys/norm + fzs = fzs/norm + # compressive part + # get a different random cube for the compressive part + # so that we can target the RMS solenoidal fraction, + # instead of setting a constant solenoidal fraction everywhere. + fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) + fxc = 0.; fyc =0.; fzc = 0. + if f_solenoidal != 1.0: + fxc = np.real(kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fyc = np.real(ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + fzc = np.real(kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) + ind = np.where(k2 == 0) + fxc[ind] = 0.; fyc[ind] = 0.; fzc[ind] = 0. + # need to normalize this before applying relative weighting of solenoidal / compressive components + norm = np.sqrt(np.sum(fxc**2+fyc**2+fzc**2)) + fxc = fxc/norm + fyc = fyc/norm + fzc = fzc/norm + # back to real space + pertx = np.real(np.fft.ifftn(f_solenoidal*fxs + (1.-f_solenoidal)*fxc)) + perty = np.real(np.fft.ifftn(f_solenoidal*fys + (1.-f_solenoidal)*fyc)) + pertz = np.real(np.fft.ifftn(f_solenoidal*fzs + (1.-f_solenoidal)*fzc)) + else: + # just convert to real space + pertx = np.real(np.fft.ifftn(fx)) + perty = np.real(np.fft.ifftn(fy)) + pertz = np.real(np.fft.ifftn(fz)) + + # subtract off COM (assuming uniform density) + pertx = pertx-np.average(pertx) + perty = perty-np.average(perty) + pertz = pertz-np.average(pertz) + # scale RMS of perturbation cube to unity + pertx, perty, pertz = normalize(pertx, perty, pertz) + return pertx, perty, pertz + + +def cut_sphere(pertx, perty, pertz, rad): + # Make radial array + x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] + x = x - (n[0]-1)/2. + y = y - (n[1]-1)/2. + z = z - (n[2]-1)/2. + r2 = x**2+y**2+z**2 + # Get range of indices we want to set to zero, and those we want to keep + idx0 = r2 > (rad*n[0]/2.0)**2 + idx1 = np.logical_not(idx0) + # Zero outside the desired region + pertx[idx0] = 0.0 + perty[idx0] = 0.0 + pertz[idx0] = 0.0 + # Recompute COM velocity, and renormalize + pertx[idx1] = pertx[idx1]-np.average(pertx[idx1]) + perty[idx1] = perty[idx1]-np.average(perty[idx1]) + pertz[idx1] = pertz[idx1]-np.average(pertz[idx1]) + pertx, perty, pertz = normalize(pertx, perty, pertz) + return pertx, perty, pertz + + +def get_erot_ke_ratio(pertx, perty, pertz, rad=-1.0): + x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] + x = x - (n[0]-1)/2. + y = y - (n[1]-1)/2. + z = z - (n[2]-1)/2. + r2 = x**2+y**2+z**2 + if rad > 0: + idx0 = r2 > (rad*n[0]/2.0)**2 + r2[idx0] = 0.0 + erot_ke_ratio = (np.sum(y*pertz-z*perty)**2 + + np.sum(z*pertx-x*pertz)**2 + + np.sum(x*perty-y*pertx)**2)/(np.sum(r2)*np.product(n)) + return erot_ke_ratio + + +def plot_spectrum1D(pertx, perty, pertz): + # plot the 1D power to check the scaling. + fx = np.abs(np.fft.fftn(pertx)) + fy = np.abs(np.fft.fftn(perty)) + fz = np.abs(np.fft.fftn(pertz)) + fx = np.abs(fx) + fy = np.abs(fy) + fz = np.abs(fz) + kx = np.zeros(n, dtype=dtype) + ky = np.zeros(n, dtype=dtype) + kz = np.zeros(n, dtype=dtype) + # perform fft k-ordering convention shifts + for j in range(0,n[1]): + for k in range(0,n[2]): + kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) + for i in range(0,n[0]): + for k in range(0,n[2]): + ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) + for i in range(0,n[0]): + for j in range(0,n[1]): + kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) + k = np.sqrt(np.array(kx**2+ky**2+kz**2,dtype=dtype)) + k1d = [] + power = [] + for i in range(kmin,kmax+1): + slice_inds = np.where(np.logical_and(k >= i, k < i+1)) + k1d.append(i+0.5) + power.append(np.sum(fx[slice_inds]**2 + fy[slice_inds]**2 + fz[slice_inds]**2)) + print(i,power[-1]) + import matplotlib.pyplot as plt + plt.loglog(k1d, power) + plt.show() + + +################### +# input parameters, read from command line +################### +parser = OptionParser() +parser.add_option('--kmin', dest='kmin', + help='minimum wavenumber.', + default=-1) +parser.add_option('--kmax', dest='kmax', + help='maximum wavenumber.', + default=-1) +parser.add_option('--size', dest='size', + help='size of each direction of data cube. default=256', + default=256) +parser.add_option('--alpha', dest='alpha', + help='negative of power law slope. (Power ~ k^-alpha) '+ + 'supersonic turbulence is near alpha=2. '+ + 'driving over a narrow band of two modes is often done with alpha=0', + default = None) +parser.add_option('--seed', dest='seed', + help='seed for random # generation. default=0', + default = 0) +parser.add_option('--f_solenoidal', dest='f_solenoidal', + help='volume RMS fraction of solenoidal component of the perturbations relative to the total. ' + + 'If --f_solenoidal=None, the motions are purely random. For low wave numbers ' + + 'the relative importance of solenoidal to compressive may be sensitive to the ' + + 'choice of random seed. It has been suggested (Federrath 2008) that ' + + 'f_solenoidal=2/3 is the most natural driving mode and this is currently' + + 'the suggested best-practice.', + default = -1) +parser.add_option('--sphererad', dest='rad', + help='if set, perturbations are set to zero outside spherical region, '+ + 'and the perturbation field is shifted and renormalized to keep the '+ + 'center of mass velocity at zero and the variance at unity; the '+ + 'spherical region cut out is centered at the center of the perturbation '+ + 'cube, and has a radius given by the value of this parameter, with sphererad = '+ + '1 corresponding to the spherical region going all the way to the edge of the '+ + 'perturbation cube', + default = -1.0) + +(options, args) = parser.parse_args() + +# size of the data domain +n = [int(options.size), int(options.size), int(options.size)] +# range of perturbation length scale in units of the smallest side of the domain +kmin = int(options.kmin) +kmax = int(options.kmax) +print(kmin, kmax) +if kmin > kmax or kmin < 0 or kmax < 0: + print("kmin must be < kmax, with kmin > 0, kmax > 0. See --help.") + sys.exit(0) +if kmax > floor(np.min(n))/2: + print("kmax must be <= floor(size/2). See --help.") + sys.exit(0) +f_solenoidal = options.f_solenoidal +if f_solenoidal == "None" or f_solenoidal == "none": + f_solenoidal = None +else: + f_solenoidal = float(options.f_solenoidal) + if f_solenoidal > 1. or f_solenoidal < 0.: + print("You must choose f_solenoidal. See --help.") + sys.exit(0) +alpha = options.alpha +if alpha==None: + print("You must choose a power law slope, alpha. See --help.") + sys.exit(0) +alpha = float(options.alpha) +if alpha < 0.: + print("alpha is less than zero. That's probably not what you want. See --help.") + sys.exit(0) +seed = int(options.seed) +# data precision +dtype = np.float64 +# ratio of solenoidal to compressive components +if options.f_solenoidal=="None" or options.f_solenoidal==None: + f_solenoidal = None +else: + f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) +rad = float(options.rad) +if rad > 1: + raise ValueError('sphererad is '+options.rad+', must be from 0 to 1') + +################### +# begin computation +################### + +np.random.seed(seed=seed) +pertx, perty, pertz = make_perturbations(n, kmin, kmax, f_solenoidal) +if rad > 0: + pertx, perty, pertz = cut_sphere(pertx, perty, pertz, rad) +erot_ke_ratio = get_erot_ke_ratio(pertx, perty, pertz, rad) +print("erot_ke_ratio = ", erot_ke_ratio) + +# hdf5 output +f = h5py.File('zdrv.hdf5', 'w') +ds = f['/'].create_dataset('pertx', n, dtype=np.float64) +ds[:] = pertx +ds = f['/'].create_dataset('perty', n, dtype=np.float64) +ds[:] = perty +ds = f['/'].create_dataset('pertz', n, dtype=np.float64) +ds[:] = pertz +f['/'].attrs['kmin'] = kmin +f['/'].attrs['kmax'] = kmax +f['/'].attrs['alpha'] = alpha +if f_solenoidal!=None: f['/'].attrs['f_solenoidal'] = f_solenoidal +if rad > 0: f['/'].attrs['sphererad'] = rad +f['/'].attrs['erot_ke_ratio'] = erot_ke_ratio +f['/'].attrs['seed'] = seed +f.close() + From 83dc726239ad76471486c32d1368e0b50f2ca12e Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sat, 3 Feb 2024 19:25:19 +0100 Subject: [PATCH 132/165] updated cmakelists --- src/StarCluster/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/StarCluster/CMakeLists.txt b/src/StarCluster/CMakeLists.txt index 277525261..796f1f1c8 100644 --- a/src/StarCluster/CMakeLists.txt +++ b/src/StarCluster/CMakeLists.txt @@ -1,7 +1,7 @@ if (AMReX_SPACEDIM EQUAL 3) # Define a custom target that runs the Python script to produce the input perturbations file - add_executable(star_cluster star_cluster.cpp TurbDataReader.cpp ${QuokkaObjSources}) + add_executable(star_cluster star_cluster.cpp ../TurbDataReader.cpp ${QuokkaObjSources}) if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(star_cluster) endif() @@ -12,7 +12,7 @@ if (AMReX_SPACEDIM EQUAL 3) OUTPUT_QUIET ) - add_test(NAME ComputeStarClusterPerturbations COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) + add_test(NAME ComputeStarClusterPerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) add_test(NAME StarCluster COMMAND star_cluster StarCluster.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests) set_tests_properties(ComputeStarClusterPerturbations PROPERTIES FIXTURES_SETUP StarCluster_fixture) set_tests_properties(StarCluster PROPERTIES FIXTURES_REQUIRED StarCluster_fixture) From ebb4c3dc87fd19b32b150bdf33b64dfe40986021 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 00:55:09 +0100 Subject: [PATCH 133/165] comment print statement --- src/TurbDataReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp index 6834c17a1..db8d4b243 100644 --- a/src/TurbDataReader.cpp +++ b/src/TurbDataReader.cpp @@ -78,7 +78,7 @@ auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); auto h_table = tableData.table(); - amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; + // amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; // fill tableData for (int i = tlo[0]; i <= thi[0]; ++i) { From b7c64c76f67bc5fc40662d64a8bc1d2964835857 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 00:57:39 +0100 Subject: [PATCH 134/165] codeQL suggestion --- src/perturbation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/perturbation.py b/src/perturbation.py index 888d1756c..b9a0f2717 100644 --- a/src/perturbation.py +++ b/src/perturbation.py @@ -262,7 +262,7 @@ def plot_spectrum1D(pertx, perty, pertz): print("You must choose f_solenoidal. See --help.") sys.exit(0) alpha = options.alpha -if alpha==None: +if alpha is None: print("You must choose a power law slope, alpha. See --help.") sys.exit(0) alpha = float(options.alpha) @@ -273,7 +273,7 @@ def plot_spectrum1D(pertx, perty, pertz): # data precision dtype = np.float64 # ratio of solenoidal to compressive components -if options.f_solenoidal=="None" or options.f_solenoidal==None: +if options.f_solenoidal is None or options.f_solenoidal is None: f_solenoidal = None else: f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) From 421546b661f6370c50965eb2ccebbd93582bf3a3 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 01:18:19 +0100 Subject: [PATCH 135/165] moved turbdatareader --- src/StarCluster/TurbDataReader.cpp | 119 ----------------------------- src/StarCluster/TurbDataReader.hpp | 59 -------------- 2 files changed, 178 deletions(-) delete mode 100644 src/StarCluster/TurbDataReader.cpp delete mode 100644 src/StarCluster/TurbDataReader.hpp diff --git a/src/StarCluster/TurbDataReader.cpp b/src/StarCluster/TurbDataReader.cpp deleted file mode 100644 index 6834c17a1..000000000 --- a/src/StarCluster/TurbDataReader.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file TurbDataReader.cpp -/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. -/// - -#include "TurbDataReader.hpp" -#include "AMReX_Arena.H" -#include "AMReX_BLassert.H" -#include "AMReX_Print.H" -#include "AMReX_TableData.H" -#include - -auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D -{ - // open dataset - hid_t dset_id = 0; - dset_id = H5Dopen2(file_id, dataset_name, H5P_DEFAULT); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dset_id != -1, "Can't open table!"); - - // get dimensions - hid_t const dspace = H5Dget_space(dset_id); - const int ndims = H5Sget_simple_extent_ndims(dspace); - std::vector dims(ndims); - H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); - - uint64_t data_size = 1; - for (int idim = 0; idim < ndims; ++idim) { - data_size *= dims[idim]; - } - - // allocate array for dataset storage - auto *temp_data = new double[data_size]; - - // read dataset - herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); - - // close dataset - status = H5Dclose(dset_id); - - // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 - // tables use row-major (C-order) indexing! - amrex::GpuArray const lo{0, 0, 0}; - amrex::GpuArray const hi{static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])}; - auto table = amrex::Table3D(temp_data, lo, hi); - return table; -} - -void initialize_turbdata(turb_data &data, std::string &data_file) -{ - amrex::Print() << "Initializing turbulence data...\n"; - amrex::Print() << fmt::format("data_file: {}.\n", data_file); - - herr_t status = 0; - herr_t const h5_error = -1; - - // open file - hid_t file_id = 0; - file_id = H5Fopen(data_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(file_id != h5_error, "Failed to open data file!"); - - data.dvx = read_dataset(file_id, "pertx"); - data.dvy = read_dataset(file_id, "perty"); - data.dvz = read_dataset(file_id, "pertz"); - - // close file - status = H5Fclose(file_id); -} - -auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData -{ - amrex::Array tlo{in_t.begin[0], in_t.begin[1], in_t.begin[2]}; - amrex::Array thi{in_t.end[0] - 1, in_t.end[1] - 1, in_t.end[2] - 1}; - amrex::TableData tableData(tlo, thi, amrex::The_Pinned_Arena()); - auto h_table = tableData.table(); - - amrex::Print() << "Copying tableData on indices " << tlo << " to " << thi << ".\n"; - - // fill tableData - for (int i = tlo[0]; i <= thi[0]; ++i) { - for (int j = tlo[1]; j <= thi[1]; ++j) { - for (int k = tlo[2]; k <= thi[2]; ++k) { - h_table(i, j, k) = in_t(i, j, k); - } - } - } - - return tableData; -} - -auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real -{ - amrex::Array tlo = dvx.lo(); - amrex::Array thi = dvx.hi(); - auto const &dvx_table = dvx.const_table(); - auto const &dvy_table = dvy.const_table(); - auto const &dvz_table = dvz.const_table(); - - // compute rms power - amrex::Real rms_sq = 0; - amrex::Long N = 0; - for (int i = tlo[0]; i <= thi[0]; ++i) { - for (int j = tlo[1]; j <= thi[1]; ++j) { - for (int k = tlo[2]; k <= thi[2]; ++k) { - amrex::Real const vx = dvx_table(i, j, k); - amrex::Real const vy = dvy_table(i, j, k); - amrex::Real const vz = dvz_table(i, j, k); - rms_sq += vx * vx + vy * vy + vz * vz; - ++N; - } - } - } - rms_sq /= static_cast(N); - return std::sqrt(rms_sq); -} diff --git a/src/StarCluster/TurbDataReader.hpp b/src/StarCluster/TurbDataReader.hpp deleted file mode 100644 index b127cc92a..000000000 --- a/src/StarCluster/TurbDataReader.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef TURBDATAREADER_HPP_ // NOLINT -#define TURBDATAREADER_HPP_ -//============================================================================== -// TwoMomentRad - a radiation transport library for patch-based AMR codes -// Copyright 2020 Benjamin Wibking. -// Released under the MIT license. See LICENSE file included in the GitHub repo. -//============================================================================== -/// \file TurbDataReader.hpp -/// \brief Reads turbulent driving fields generated as cubic HDF5 arrays. -/// - -#include -#include -#include -#include -#include - -#include "fmt/core.h" -#include -#include -#include - -#include "AMReX.H" -#include "AMReX_Array.H" -#include "AMReX_BLassert.H" -#include "AMReX_TableData.H" - -/* HDF5 definitions */ - -#define HDF5_FILE_I4 H5T_STD_I32BE -#define HDF5_FILE_I8 H5T_STD_I64BE -#define HDF5_FILE_R4 H5T_IEEE_F32BE -#define HDF5_FILE_R8 H5T_IEEE_F64BE -#define HDF5_FILE_B8 H5T_STD_B8BE - -#define HDF5_I4 H5T_NATIVE_INT -#define HDF5_I8 H5T_NATIVE_LLONG -#define HDF5_R4 H5T_NATIVE_FLOAT -#define HDF5_R8 H5T_NATIVE_DOUBLE -#define HDF5_R16 H5T_NATIVE_LDOUBLE - -// Cooling table storage - -using turb_data = struct turb_data { - // values - amrex::Table3D dvx; - amrex::Table3D dvy; - amrex::Table3D dvz; -}; - -void initialize_turbdata(turb_data &data, std::string &data_file); - -auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D; - -auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData; - -auto computeRms(amrex::TableData &dvx, amrex::TableData &dvy, amrex::TableData &dvz) -> amrex::Real; - -#endif // TURBDATAREADER_HPP_ From e6aa3f482eda61513481327eedc6f70231de0eb1 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 01:18:37 +0100 Subject: [PATCH 136/165] clang-tidy warnings --- src/StarCluster/perturbation.py | 311 -------------------------------- src/TurbDataReader.cpp | 6 +- 2 files changed, 3 insertions(+), 314 deletions(-) delete mode 100644 src/StarCluster/perturbation.py diff --git a/src/StarCluster/perturbation.py b/src/StarCluster/perturbation.py deleted file mode 100644 index 888d1756c..000000000 --- a/src/StarCluster/perturbation.py +++ /dev/null @@ -1,311 +0,0 @@ -import h5py -import numpy as np -import numpy.fft -from math import * -from optparse import OptionParser -import sys - - -def init_perturbations(n, kmin, kmax, dtype): - kx = np.zeros(n, dtype=dtype) - ky = np.zeros(n, dtype=dtype) - kz = np.zeros(n, dtype=dtype) - # perform fft k-ordering convention shifts - for j in range(0,n[1]): - for k in range(0,n[2]): - kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) - for i in range(0,n[0]): - for k in range(0,n[2]): - ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) - for i in range(0,n[0]): - for j in range(0,n[1]): - kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) - - kx = np.array(kx, dtype=dtype) - ky = np.array(ky, dtype=dtype) - kz = np.array(kz, dtype=dtype) - k = np.sqrt(np.array(kx**2+ky**2+kz**2, dtype=dtype)) - - # only use the positive frequencies - inds = np.where(np.logical_and(k**2 >= kmin**2, k**2 < (kmax+1)**2)) - nr = len(inds[0]) - - phasex = np.zeros(n, dtype=dtype) - phasex[inds] = 2.*pi*np.random.uniform(size=nr) - fx = np.zeros(n, dtype=dtype) - fx[inds] = np.random.normal(size=nr) - - phasey = np.zeros(n, dtype=dtype) - phasey[inds] = 2.*pi*np.random.uniform(size=nr) - fy = np.zeros(n, dtype=dtype) - fy[inds] = np.random.normal(size=nr) - - phasez = np.zeros(n, dtype=dtype) - phasez[inds] = 2.*pi*np.random.uniform(size=nr) - fz = np.zeros(n, dtype=dtype) - fz[inds] = np.random.normal(size=nr) - - # rescale perturbation amplitude so that low number statistics - # at low k do not throw off the desired power law scaling. - for i in range(kmin, kmax+1): - slice_inds = np.where(np.logical_and(k >= i, k < i+1)) - rescale = sqrt(np.sum(np.abs(fx[slice_inds])**2 + np.abs(fy[slice_inds])**2 + np.abs(fz[slice_inds])**2)) - fx[slice_inds] = fx[slice_inds]/rescale - fy[slice_inds] = fy[slice_inds]/rescale - fz[slice_inds] = fz[slice_inds]/rescale - - # set the power law behavior - # wave number bins - fx[inds] = fx[inds]*k[inds]**-(0.5*alpha) - fy[inds] = fy[inds]*k[inds]**-(0.5*alpha) - fz[inds] = fz[inds]*k[inds]**-(0.5*alpha) - - # add in phases - fx = np.cos(phasex)*fx + 1j*np.sin(phasex)*fx - fy = np.cos(phasey)*fy + 1j*np.sin(phasey)*fy - fz = np.cos(phasez)*fz + 1j*np.sin(phasez)*fz - - return fx, fy, fz, kx, ky, kz - - -def normalize(fx, fy, fz): - norm = np.sqrt(np.sum(fx**2 + fy**2 + fz**2)/np.product(n)) - fx = fx/norm - fy = fy/norm - fz = fz/norm - return fx, fy, fz - - -def make_perturbations(n, kmin, kmax, f_solenoidal): - fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) - if f_solenoidal != None: - k2 = kx**2+ky**2+kz**2 - # solenoidal part - fxs = 0.; fys =0.; fzs = 0. - if f_solenoidal != 0.0: - fxs = np.real(fx - kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fys = np.real(fy - ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fzs = np.real(fz - kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - ind = np.where(k2 == 0) - fxs[ind] = 0.; fys[ind] = 0.; fzs[ind] = 0. - # need to normalize this before applying relative weighting of solenoidal / compressive components - norm = np.sqrt(np.sum(fxs**2+fys**2+fzs**2)) - fxs = fxs/norm - fys = fys/norm - fzs = fzs/norm - # compressive part - # get a different random cube for the compressive part - # so that we can target the RMS solenoidal fraction, - # instead of setting a constant solenoidal fraction everywhere. - fx, fy, fz, kx, ky, kz = init_perturbations(n, kmin, kmax, dtype) - fxc = 0.; fyc =0.; fzc = 0. - if f_solenoidal != 1.0: - fxc = np.real(kx*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fyc = np.real(ky*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - fzc = np.real(kz*(kx*fx+ky*fy+kz*fz)/np.maximum(k2,1e-16)) - ind = np.where(k2 == 0) - fxc[ind] = 0.; fyc[ind] = 0.; fzc[ind] = 0. - # need to normalize this before applying relative weighting of solenoidal / compressive components - norm = np.sqrt(np.sum(fxc**2+fyc**2+fzc**2)) - fxc = fxc/norm - fyc = fyc/norm - fzc = fzc/norm - # back to real space - pertx = np.real(np.fft.ifftn(f_solenoidal*fxs + (1.-f_solenoidal)*fxc)) - perty = np.real(np.fft.ifftn(f_solenoidal*fys + (1.-f_solenoidal)*fyc)) - pertz = np.real(np.fft.ifftn(f_solenoidal*fzs + (1.-f_solenoidal)*fzc)) - else: - # just convert to real space - pertx = np.real(np.fft.ifftn(fx)) - perty = np.real(np.fft.ifftn(fy)) - pertz = np.real(np.fft.ifftn(fz)) - - # subtract off COM (assuming uniform density) - pertx = pertx-np.average(pertx) - perty = perty-np.average(perty) - pertz = pertz-np.average(pertz) - # scale RMS of perturbation cube to unity - pertx, perty, pertz = normalize(pertx, perty, pertz) - return pertx, perty, pertz - - -def cut_sphere(pertx, perty, pertz, rad): - # Make radial array - x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] - x = x - (n[0]-1)/2. - y = y - (n[1]-1)/2. - z = z - (n[2]-1)/2. - r2 = x**2+y**2+z**2 - # Get range of indices we want to set to zero, and those we want to keep - idx0 = r2 > (rad*n[0]/2.0)**2 - idx1 = np.logical_not(idx0) - # Zero outside the desired region - pertx[idx0] = 0.0 - perty[idx0] = 0.0 - pertz[idx0] = 0.0 - # Recompute COM velocity, and renormalize - pertx[idx1] = pertx[idx1]-np.average(pertx[idx1]) - perty[idx1] = perty[idx1]-np.average(perty[idx1]) - pertz[idx1] = pertz[idx1]-np.average(pertz[idx1]) - pertx, perty, pertz = normalize(pertx, perty, pertz) - return pertx, perty, pertz - - -def get_erot_ke_ratio(pertx, perty, pertz, rad=-1.0): - x, y, z = np.mgrid[0:n[0], 0:n[1], 0:n[2]] - x = x - (n[0]-1)/2. - y = y - (n[1]-1)/2. - z = z - (n[2]-1)/2. - r2 = x**2+y**2+z**2 - if rad > 0: - idx0 = r2 > (rad*n[0]/2.0)**2 - r2[idx0] = 0.0 - erot_ke_ratio = (np.sum(y*pertz-z*perty)**2 + - np.sum(z*pertx-x*pertz)**2 + - np.sum(x*perty-y*pertx)**2)/(np.sum(r2)*np.product(n)) - return erot_ke_ratio - - -def plot_spectrum1D(pertx, perty, pertz): - # plot the 1D power to check the scaling. - fx = np.abs(np.fft.fftn(pertx)) - fy = np.abs(np.fft.fftn(perty)) - fz = np.abs(np.fft.fftn(pertz)) - fx = np.abs(fx) - fy = np.abs(fy) - fz = np.abs(fz) - kx = np.zeros(n, dtype=dtype) - ky = np.zeros(n, dtype=dtype) - kz = np.zeros(n, dtype=dtype) - # perform fft k-ordering convention shifts - for j in range(0,n[1]): - for k in range(0,n[2]): - kx[:,j,k] = n[0]*np.fft.fftfreq(n[0]) - for i in range(0,n[0]): - for k in range(0,n[2]): - ky[i,:,k] = n[1]*np.fft.fftfreq(n[1]) - for i in range(0,n[0]): - for j in range(0,n[1]): - kz[i,j,:] = n[2]*np.fft.fftfreq(n[2]) - k = np.sqrt(np.array(kx**2+ky**2+kz**2,dtype=dtype)) - k1d = [] - power = [] - for i in range(kmin,kmax+1): - slice_inds = np.where(np.logical_and(k >= i, k < i+1)) - k1d.append(i+0.5) - power.append(np.sum(fx[slice_inds]**2 + fy[slice_inds]**2 + fz[slice_inds]**2)) - print(i,power[-1]) - import matplotlib.pyplot as plt - plt.loglog(k1d, power) - plt.show() - - -################### -# input parameters, read from command line -################### -parser = OptionParser() -parser.add_option('--kmin', dest='kmin', - help='minimum wavenumber.', - default=-1) -parser.add_option('--kmax', dest='kmax', - help='maximum wavenumber.', - default=-1) -parser.add_option('--size', dest='size', - help='size of each direction of data cube. default=256', - default=256) -parser.add_option('--alpha', dest='alpha', - help='negative of power law slope. (Power ~ k^-alpha) '+ - 'supersonic turbulence is near alpha=2. '+ - 'driving over a narrow band of two modes is often done with alpha=0', - default = None) -parser.add_option('--seed', dest='seed', - help='seed for random # generation. default=0', - default = 0) -parser.add_option('--f_solenoidal', dest='f_solenoidal', - help='volume RMS fraction of solenoidal component of the perturbations relative to the total. ' + - 'If --f_solenoidal=None, the motions are purely random. For low wave numbers ' + - 'the relative importance of solenoidal to compressive may be sensitive to the ' + - 'choice of random seed. It has been suggested (Federrath 2008) that ' + - 'f_solenoidal=2/3 is the most natural driving mode and this is currently' + - 'the suggested best-practice.', - default = -1) -parser.add_option('--sphererad', dest='rad', - help='if set, perturbations are set to zero outside spherical region, '+ - 'and the perturbation field is shifted and renormalized to keep the '+ - 'center of mass velocity at zero and the variance at unity; the '+ - 'spherical region cut out is centered at the center of the perturbation '+ - 'cube, and has a radius given by the value of this parameter, with sphererad = '+ - '1 corresponding to the spherical region going all the way to the edge of the '+ - 'perturbation cube', - default = -1.0) - -(options, args) = parser.parse_args() - -# size of the data domain -n = [int(options.size), int(options.size), int(options.size)] -# range of perturbation length scale in units of the smallest side of the domain -kmin = int(options.kmin) -kmax = int(options.kmax) -print(kmin, kmax) -if kmin > kmax or kmin < 0 or kmax < 0: - print("kmin must be < kmax, with kmin > 0, kmax > 0. See --help.") - sys.exit(0) -if kmax > floor(np.min(n))/2: - print("kmax must be <= floor(size/2). See --help.") - sys.exit(0) -f_solenoidal = options.f_solenoidal -if f_solenoidal == "None" or f_solenoidal == "none": - f_solenoidal = None -else: - f_solenoidal = float(options.f_solenoidal) - if f_solenoidal > 1. or f_solenoidal < 0.: - print("You must choose f_solenoidal. See --help.") - sys.exit(0) -alpha = options.alpha -if alpha==None: - print("You must choose a power law slope, alpha. See --help.") - sys.exit(0) -alpha = float(options.alpha) -if alpha < 0.: - print("alpha is less than zero. That's probably not what you want. See --help.") - sys.exit(0) -seed = int(options.seed) -# data precision -dtype = np.float64 -# ratio of solenoidal to compressive components -if options.f_solenoidal=="None" or options.f_solenoidal==None: - f_solenoidal = None -else: - f_solenoidal = min(max(float(options.f_solenoidal), 0.), 1.) -rad = float(options.rad) -if rad > 1: - raise ValueError('sphererad is '+options.rad+', must be from 0 to 1') - -################### -# begin computation -################### - -np.random.seed(seed=seed) -pertx, perty, pertz = make_perturbations(n, kmin, kmax, f_solenoidal) -if rad > 0: - pertx, perty, pertz = cut_sphere(pertx, perty, pertz, rad) -erot_ke_ratio = get_erot_ke_ratio(pertx, perty, pertz, rad) -print("erot_ke_ratio = ", erot_ke_ratio) - -# hdf5 output -f = h5py.File('zdrv.hdf5', 'w') -ds = f['/'].create_dataset('pertx', n, dtype=np.float64) -ds[:] = pertx -ds = f['/'].create_dataset('perty', n, dtype=np.float64) -ds[:] = perty -ds = f['/'].create_dataset('pertz', n, dtype=np.float64) -ds[:] = pertz -f['/'].attrs['kmin'] = kmin -f['/'].attrs['kmax'] = kmax -f['/'].attrs['alpha'] = alpha -if f_solenoidal!=None: f['/'].attrs['f_solenoidal'] = f_solenoidal -if rad > 0: f['/'].attrs['sphererad'] = rad -f['/'].attrs['erot_ke_ratio'] = erot_ke_ratio -f['/'].attrs['seed'] = seed -f.close() - diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp index db8d4b243..f9687b579 100644 --- a/src/TurbDataReader.cpp +++ b/src/TurbDataReader.cpp @@ -33,14 +33,14 @@ auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D temp_data(new double[data_size]); // read dataset herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(status != -1, "Failed to read dataset!"); // close dataset - status = H5Dclose(dset_id); + H5Dclose(dset_id); // WARNING: Table3D uses column-major (Fortran-order) indexing, but HDF5 // tables use row-major (C-order) indexing! @@ -68,7 +68,7 @@ void initialize_turbdata(turb_data &data, std::string &data_file) data.dvz = read_dataset(file_id, "pertz"); // close file - status = H5Fclose(file_id); + H5Fclose(file_id); } auto get_tabledata(amrex::Table3D &in_t) -> amrex::TableData From 69616f88ece93869e50e703238ace0ee4cf54c04 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 11:15:19 +0100 Subject: [PATCH 137/165] revert --- .github/workflows/clang-tidy.yml | 1 - src/PrimordialChem/test_primordial_chem.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 38a2296e7..3e2fd32e8 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -25,7 +25,6 @@ jobs: apt_packages: libopenmpi-dev,libhdf5-mpi-dev,python3-dev,python3-numpy,python3-matplotlib cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF split_workflow: true - extra_args: '-- src/PrimordialChem/test_primordial_chem.cpp' # Uploads an artefact containing clang_fixes.json - uses: ZedThree/clang-tidy-review/upload@v0.17.1 diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index c4cc467cd..8e6409dd6 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -38,7 +38,7 @@ using amrex::Real; struct PrimordialChemTest { -}; // dummy type to allow compile-type polymorphism via template specializationnnnnn +}; // dummy type to allow compile-type polymorphism via template specialization template <> struct Physics_Traits { // cell-centred From 702faa429ac4d722f47968a3fe3a149169770956 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 11:43:43 +0100 Subject: [PATCH 138/165] reorder includes --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 8e6409dd6..7772b07e4 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,11 +29,11 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" #include "network.H" +#include "actual_eos_data.H" using amrex::Real; From d953142d37cc3eed6a4ac26dcc4e4d63bbd7c15c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 10:43:58 +0000 Subject: [PATCH 139/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 7772b07e4..8e6409dd6 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,11 +29,11 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" +#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" #include "network.H" -#include "actual_eos_data.H" using amrex::Real; From 03eb8d9ac668ba72c67066990f905dc8cb63e798 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 11:45:55 +0100 Subject: [PATCH 140/165] debug --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 8e6409dd6..401ad4a9a 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,7 +29,7 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include "actual_eos_data.H" +#include"actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From b78f5636dc50d947f343460c3671ae4e3f73776e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 10:46:39 +0000 Subject: [PATCH 141/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 401ad4a9a..8e6409dd6 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,7 +29,7 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include"actual_eos_data.H" +#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From 75750607594d07b23b0ae05916d465ff68484146 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 11:50:16 +0100 Subject: [PATCH 142/165] turn off pre-commit --- .pre-commit-config.yaml | 21 +++++++++++---------- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7620f9726..ed0274bc1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,14 +14,15 @@ repos: rev: v17.0.6 hooks: - id: clang-format -ci: - autofix_commit_msg: | - [pre-commit.ci] auto fixes from pre-commit.com hooks +# ci section is removed or commented out +# ci: +# autofix_commit_msg: | +# [pre-commit.ci] auto fixes from pre-commit.com hooks +# for more information, see https://pre-commit.ci +# autofix_prs: true +# autoupdate_branch: '' +# autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' +# autoupdate_schedule: weekly +# skip: [] +# submodules: false - for more information, see https://pre-commit.ci - autofix_prs: true - autoupdate_branch: '' - autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' - autoupdate_schedule: weekly - skip: [] - submodules: false \ No newline at end of file diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 8e6409dd6..401ad4a9a 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,7 +29,7 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include "actual_eos_data.H" +#include"actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From da515d6a6cc81d6acf240788172d1c091b117969 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 10:50:34 +0000 Subject: [PATCH 143/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 401ad4a9a..8e6409dd6 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,7 +29,7 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include"actual_eos_data.H" +#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From dfe52d172e917c5d79520678ebb5473b5d70b133 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 11:51:25 +0100 Subject: [PATCH 144/165] remove precommit --- .pre-commit-config.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index ed0274bc1..000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: check-yaml - - id: check-added-large-files - - id: check-case-conflict - - id: check-merge-conflict - - id: check-symlinks - - id: mixed-line-ending -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: v17.0.6 - hooks: - - id: clang-format -# ci section is removed or commented out -# ci: -# autofix_commit_msg: | -# [pre-commit.ci] auto fixes from pre-commit.com hooks -# for more information, see https://pre-commit.ci -# autofix_prs: true -# autoupdate_branch: '' -# autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' -# autoupdate_schedule: weekly -# skip: [] -# submodules: false - From 87c4fb49d00c885509eaab7e37b7c1813809832c Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 11:53:07 +0100 Subject: [PATCH 145/165] debug --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 8e6409dd6..401ad4a9a 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,7 +29,7 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include "actual_eos_data.H" +#include"actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From dfa1c90a5e348fa606146479e17d865f3b7c2eac Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 13:15:23 +0100 Subject: [PATCH 146/165] dont build primordial chem --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cc24e6ecd..12a623349 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -136,7 +136,7 @@ add_subdirectory(PopIII) add_subdirectory(StarCluster) -add_subdirectory(PrimordialChem) +#add_subdirectory(PrimordialChem) add_subdirectory(Advection) add_subdirectory(Advection2D) From e0d99269b243e897f2589182d4807cd414afdad8 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 13:21:08 +0100 Subject: [PATCH 147/165] revert --- src/PrimordialChem/test_primordial_chem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PrimordialChem/test_primordial_chem.cpp b/src/PrimordialChem/test_primordial_chem.cpp index 401ad4a9a..8e6409dd6 100644 --- a/src/PrimordialChem/test_primordial_chem.cpp +++ b/src/PrimordialChem/test_primordial_chem.cpp @@ -29,7 +29,7 @@ #include "radiation_system.hpp" #include "test_primordial_chem.hpp" -#include"actual_eos_data.H" +#include "actual_eos_data.H" #include "burn_type.H" #include "eos.H" #include "extern_parameters.H" From c2ccaea6789edbcb1e16f41b6418406a2c29f4f8 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 14:25:26 +0100 Subject: [PATCH 148/165] cmake in 3D --- .github/workflows/clang-tidy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 3e2fd32e8..13994d366 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -23,7 +23,7 @@ jobs: config_file: src/.clang-tidy build_dir: build apt_packages: libopenmpi-dev,libhdf5-mpi-dev,python3-dev,python3-numpy,python3-matplotlib - cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF + cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF -DAMReX_SPACEDIM=3 split_workflow: true # Uploads an artefact containing clang_fixes.json From 38963af141ae4a32ad4293082720aeeb17f6d693 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 14:48:28 +0100 Subject: [PATCH 149/165] fix clang-tidy warnings --- src/PopIII/popiii.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index fe69cfdfc..9d5acf041 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -343,10 +343,6 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra auto const &prob_lo = geom[lev].ProbLoArray(); auto const &prob_hi = geom[lev].ProbHiArray(); - amrex::Real const x0 = prob_lo[0] + 0.5 * (prob_hi[0] - prob_lo[0]); - amrex::Real const y0 = prob_lo[1] + 0.5 * (prob_hi[1] - prob_lo[1]); - amrex::Real const z0 = prob_lo[2] + 0.5 * (prob_hi[2] - prob_lo[2]); - for (amrex::MFIter mfi(state_new_cc_[lev]); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.validbox(); const auto state = state_new_cc_[lev].const_array(mfi); @@ -362,7 +358,7 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra Real const pressure = HydroSystem::ComputePressure(state, i, j, k); amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state, i, j, k); - amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + amrex::Real const cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); if (l_Jeans < (N_cells * dx) && rho > 2e-20) { @@ -382,7 +378,7 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { Real const rho = state[bx](i, j, k, HydroSystem::density_index); - amrex::Real Eint = state[bx](i, j, k, HydroSystem::internalEnergy_index); + amrex::Real const Eint = state[bx](i, j, k, HydroSystem::internalEnergy_index); amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); @@ -397,7 +393,7 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str auto output = mf.arrays(); amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { - amrex::Real Pgas = HydroSystem::ComputePressure(state[bx], i, j, k); + amrex::Real const Pgas = HydroSystem::ComputePressure(state[bx], i, j, k); output[bx](i, j, k, ncomp) = Pgas; }); } @@ -423,10 +419,10 @@ template <> void RadhydroSimulation::ComputeDerivedVar(int lev, std::str amrex::ParallelFor(mf, [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { Real const rho = state[bx](i, j, k, HydroSystem::density_index); - Real pressure = HydroSystem::ComputePressure(state[bx], i, j, k); + Real const pressure = HydroSystem::ComputePressure(state[bx], i, j, k); amrex::GpuArray::numMassScalars> massScalars = RadSystem::ComputeMassScalars(state[bx], i, j, k); - amrex::Real cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); + amrex::Real const cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); output[bx](i, j, k, ncomp) = cs; }); } From d2f1377510876595b5cc423c462cd650366703dc Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 15:08:46 +0100 Subject: [PATCH 150/165] readded precommit --- .pre-commit-config.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..036211169 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: mixed-line-ending +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: v17.0.6 + hooks: + - id: clang-format +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_branch: '' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false From 0dd17b3ab453084a43aef5d1a9068fb2ab965edb Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 21:28:00 +0100 Subject: [PATCH 151/165] trying to fix table3d error --- src/TurbDataReader.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp index f9687b579..bddf1a714 100644 --- a/src/TurbDataReader.cpp +++ b/src/TurbDataReader.cpp @@ -27,13 +27,14 @@ auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D dims(ndims); H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); - uint64_t data_size = 1; + size_t data_size = 1; for (int idim = 0; idim < ndims; ++idim) { data_size *= dims[idim]; } // allocate array for dataset storage - std::unique_ptr temp_data(new double[data_size]); + // std::unique_ptr temp_data(new double[data_size]); + std::vector temp_data(data_size); // read dataset herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); From 91e112a256d2b3074cd4f00eccfb018bf811852e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 20:28:14 +0000 Subject: [PATCH 152/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/TurbDataReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp index bddf1a714..58123f7a5 100644 --- a/src/TurbDataReader.cpp +++ b/src/TurbDataReader.cpp @@ -27,7 +27,7 @@ auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D dims(ndims); H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); - size_t data_size = 1; + size_t data_size = 1; for (int idim = 0; idim < ndims; ++idim) { data_size *= dims[idim]; } From eb14f5808b4068c12d7a6b644b2c1945b6d39ff9 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Sun, 4 Feb 2024 21:29:22 +0100 Subject: [PATCH 153/165] revert --- src/TurbDataReader.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/TurbDataReader.cpp b/src/TurbDataReader.cpp index 58123f7a5..cb5300b68 100644 --- a/src/TurbDataReader.cpp +++ b/src/TurbDataReader.cpp @@ -27,14 +27,13 @@ auto read_dataset(hid_t &file_id, char const *dataset_name) -> amrex::Table3D dims(ndims); H5Sget_simple_extent_dims(dspace, dims.data(), nullptr); - size_t data_size = 1; + uint64_t data_size = 1; for (int idim = 0; idim < ndims; ++idim) { data_size *= dims[idim]; } // allocate array for dataset storage - // std::unique_ptr temp_data(new double[data_size]); - std::vector temp_data(data_size); + auto *temp_data = new double[data_size]; // read dataset herr_t status = H5Dread(dset_id, HDF5_R8, H5S_ALL, H5S_ALL, H5P_DEFAULT, temp_data); From a403431e2c2aa0b7efd2a9ee35a0971f4ece0b06 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 10:50:45 +0100 Subject: [PATCH 154/165] arrange tests alphabetically --- src/CMakeLists.txt | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 12a623349..91e394b1f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -132,12 +132,6 @@ set (QuokkaObjSources "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp" "${CMAKE_CURRENT_SO #endif() #link_libraries(QuokkaObj) -add_subdirectory(PopIII) - - -add_subdirectory(StarCluster) -#add_subdirectory(PrimordialChem) - add_subdirectory(Advection) add_subdirectory(Advection2D) add_subdirectory(AdvectionSemiellipse) @@ -180,11 +174,14 @@ add_subdirectory(RadhydroUniformAdvecting) add_subdirectory(RadhydroPulse) add_subdirectory(RadhydroPulseMG) -add_subdirectory(ODEIntegration) +add_subdirectory(BinaryOrbitCIC) add_subdirectory(Cooling) -add_subdirectory(ShockCloud) -add_subdirectory(PassiveScalar) add_subdirectory(FCQuantities) -add_subdirectory(SphericalCollapse) add_subdirectory(NSCBC) -add_subdirectory(BinaryOrbitCIC) +add_subdirectory(ODEIntegration) +add_subdirectory(PassiveScalar) +add_subdirectory(PrimordialChem) +add_subdirectory(PopIII) +add_subdirectory(ShockCloud) +add_subdirectory(StarCluster) +add_subdirectory(SphericalCollapse) From a7557c7f30131ed414911487746b3cd94da422e6 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 10:51:10 +0100 Subject: [PATCH 155/165] reduce grid size to 128 --- tests/PopIII.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/PopIII.in b/tests/PopIII.in index 4f1b5b5a7..6515d22a9 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -16,7 +16,7 @@ amr.v = 1 # verbosity in Amr amr.n_cell = 64 64 64 amr.max_level = 0 # number of levels = max_level + 1 amr.blocking_factor = 32 # grid size must be divisible by this -amr.max_grid_size = 1024 # at least 128 for GPUs +amr.max_grid_size = 128 # at least 128 for GPUs amr.n_error_buf = 3 # minimum 3 cell buffer around tagged cells amr.grid_eff = 0.7 # default From 5959fa1a05d7130d04b51be69087261bfaaa7693 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 10:51:23 +0100 Subject: [PATCH 156/165] comment for h5py --- src/PopIII/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PopIII/CMakeLists.txt b/src/PopIII/CMakeLists.txt index 3d6f3f984..b379db66f 100644 --- a/src/PopIII/CMakeLists.txt +++ b/src/PopIII/CMakeLists.txt @@ -17,6 +17,7 @@ if (AMReX_SPACEDIM EQUAL 3) endif() #need h5py to run perturbations.py file below + #purpose of this code is to check whether the h5py python package is installed execute_process( COMMAND Python3::Interpreter -c "h5py" RESULT_VARIABLE EXIT_CODE From b2cd1af7eaef2eb542704e5aeb40581aadf1b7b2 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 10:58:06 +0100 Subject: [PATCH 157/165] make initial rms vel a runtime param --- src/PopIII/popiii.cpp | 3 ++- tests/PopIII.in | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 9d5acf041..09eeb6cbe 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -165,7 +165,8 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; - const Real rms_dv_target = 1.8050e5; + const Real rms_dv_target; + pp.query("rms_velocity", rms_dv_target); const Real rms_dv_actual = userData_.dv_rms_generated; userData_.rescale_factor = rms_dv_target / rms_dv_actual; diff --git a/tests/PopIII.in b/tests/PopIII.in index 6515d22a9..3de267fc0 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -32,9 +32,10 @@ do_subcycle = 0 plotfile_interval = 50 #100 checkpoint_interval = 200 -perturb.cloud_radius = 3.086e18 -perturb.cloud_omega = 2.016008E-14 -perturb.cloud_numdens = 0.90861183E+004 +perturb.cloud_radius = 3.086e18 #initial cloud radius in cm +perturb.cloud_omega = 2.016008E-14 #initial cloud angular velocity in s^-1 +perturb.cloud_numdens = 0.90861183E+004 #initial cloud number density in cm^-3 +perturb.rms_velocity = 1.8050e5 #initial cloud rms velocity (to drive turbulence) # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/ From 4c7a7fc26237b144aa27a8d81a50877a98a6290e Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 11:17:28 +0100 Subject: [PATCH 158/165] let the code calculate pressure --- src/PopIII/popiii.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 09eeb6cbe..8bb62bdf0 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -165,7 +165,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; - const Real rms_dv_target; + Real rms_dv_target; pp.query("rms_velocity", rms_dv_target); const Real rms_dv_actual = userData_.dv_rms_generated; userData_.rescale_factor = rms_dv_target / rms_dv_actual; @@ -299,18 +299,20 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: double vy = renorm_amp * dvy(i, j, k); double const vz = renorm_amp * dvz(i, j, k); - if (r <= R_sphere) { - state.rho = rhotot; - state.T = core_temp; - eos(eos_input_rt, state); + // calculate eos params for the core + state.rho = rhotot; + state.T = core_temp; + eos(eos_input_rt, state); + if (r <= R_sphere) { // add rotation to vx and vy vx += (-1.0) * distxy * omega_sphere * std::sin(phi); vy += distxy * omega_sphere * std::cos(phi); } else { + // re-calculate eos params outside the core, using pressure equilibrium (so, pressure within the core = pressure outside) state.rho = 0.01 * rhotot; - state.p = 3.595730e-10; // pressure equilibrium - this is the pressure within the core + state.p = state.p; eos(eos_input_rp, state); } From 47315f8454cce842ae6702fcc53e034bb06149df Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 11:26:08 +0100 Subject: [PATCH 159/165] make jeans refinement params runtime params --- src/PopIII/popiii.cpp | 14 +++++++++++--- tests/PopIII.in | 4 ++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 8bb62bdf0..5c210a1b5 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -338,8 +338,14 @@ template <> void RadhydroSimulation::setInitialConditionsOnGrid(quokka:: template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArray &tags, amrex::Real /*time*/, int /*ngrow*/) { - // refine on Jeans length - const int N_cells = 64; // inverse of the 'Jeans number' [Truelove et al. (1997)] + + // read-in jeans length refinement runtime params + amrex::ParmParse const pp("jeansRefine"); + int N_cells; + pp.query("ncells", N_cells); // inverse of the 'Jeans number' [Truelove et al. (1997)] + Real jeans_density_threshold; + pp.query("density_threshold", jeans_density_threshold); + const amrex::Real G = Gconst_; const amrex::Real dx = geom[lev].CellSizeArray()[0]; @@ -364,7 +370,9 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra amrex::Real const cs = quokka::EOS::ComputeSoundSpeed(rho, pressure, massScalars); const amrex::Real l_Jeans = cs * std::sqrt(M_PI / (G * rho)); - if (l_Jeans < (N_cells * dx) && rho > 2e-20) { + // add a density criterion for refinement so that no initial refinement is ever triggered outside the core + // typically, a density threshold ~ initial core density works well + if (l_Jeans < (N_cells * dx) && rho > jeans_density_threshold) { tag(i, j, k) = amrex::TagBox::SET; } }); diff --git a/tests/PopIII.in b/tests/PopIII.in index 3de267fc0..f7f82a147 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -37,6 +37,10 @@ perturb.cloud_omega = 2.016008E-14 #initial cloud angular velocity in s^-1 perturb.cloud_numdens = 0.90861183E+004 #initial cloud number density in cm^-3 perturb.rms_velocity = 1.8050e5 #initial cloud rms velocity (to drive turbulence) +# params for jeans refinement +jeansRefine.ncells = 64 #refine on these many cells per jeans length +jeansRefine.jeans_density_threshold = 2e-20 #do not refine if density is less than this density + # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" From e313ba7b99b75af2d5eaf5f37865514a4bd412eb Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 11:34:01 +0100 Subject: [PATCH 160/165] move dt < 0 outside of the inner loop --- src/Chemistry.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Chemistry.hpp b/src/Chemistry.hpp index da8dfdf52..31a4248aa 100644 --- a/src/Chemistry.hpp +++ b/src/Chemistry.hpp @@ -39,6 +39,10 @@ template void computeChemistry(amrex::MultiFab &mf, const R const amrex::Box &indexRange = iter.validbox(); auto const &state = mf.array(iter); + if (dt < 0) { + amrex::Abort("Cannot do chemistry with dt < 0!"); + } + amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { const Real rho = state(i, j, k, HydroSystem::density_index); const Real xmom = state(i, j, k, HydroSystem::x1Momentum_index); @@ -83,10 +87,6 @@ template void computeChemistry(amrex::MultiFab &mf, const R // call the EOS to set initial internal energy e eos(eos_input_re, chemstate); - if (dt < 0) { - amrex::Abort("Cannot do chemistry with dt < 0!"); - } - // do the actual integration // do it in .cpp so that it is not built at compile time for all tests // which would otherwise slow down compilation due to the large RHS file From 37f1ba157b570ddb3812dc6ff38d17972986df8c Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 12:01:04 +0100 Subject: [PATCH 161/165] declaring an initial value of vars --- src/PopIII/popiii.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 5c210a1b5..6c052749b 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -165,7 +165,7 @@ template <> void RadhydroSimulation::preCalculateInitialConditions() userData_.dv_rms_generated = computeRms(pinned_dvx, pinned_dvy, pinned_dvz); amrex::Print() << "rms dv = " << userData_.dv_rms_generated << "\n"; - Real rms_dv_target; + Real rms_dv_target = NAN; pp.query("rms_velocity", rms_dv_target); const Real rms_dv_actual = userData_.dv_rms_generated; userData_.rescale_factor = rms_dv_target / rms_dv_actual; @@ -341,9 +341,9 @@ template <> void RadhydroSimulation::ErrorEst(int lev, amrex::TagBoxArra // read-in jeans length refinement runtime params amrex::ParmParse const pp("jeansRefine"); - int N_cells; + int N_cells = 0; pp.query("ncells", N_cells); // inverse of the 'Jeans number' [Truelove et al. (1997)] - Real jeans_density_threshold; + Real jeans_density_threshold = NAN; pp.query("density_threshold", jeans_density_threshold); const amrex::Real G = Gconst_; From 414e0e58c8f3928172d17658b4333ac24b51ea8f Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 15:15:51 +0100 Subject: [PATCH 162/165] make density floor runtime param --- src/PopIII/popiii.cpp | 7 ++++++- tests/PopIII.in | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index 6c052749b..d7fb04a7d 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -469,7 +469,12 @@ auto problem_main() -> int // Problem initialization RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity - sim.densityFloor_ = 1e-25; + + amrex::ParmParse const pp("popiii"); + Real densfloor = NAN; + pp.query("density_floor", densfloor); + sim.densityFloor_ = densfloor; + sim.tempFloor_ = 2.73 * (30.0 + 1.0); // sim.speedCeiling_ = 3e6; diff --git a/tests/PopIII.in b/tests/PopIII.in index f7f82a147..c390b46ac 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -41,6 +41,9 @@ perturb.rms_velocity = 1.8050e5 #initial cloud rms velocity (to drive turbulence jeansRefine.ncells = 64 #refine on these many cells per jeans length jeansRefine.jeans_density_threshold = 2e-20 #do not refine if density is less than this density +# density floor for popiii +popiii.densfloor = 1e-25 + # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/ perturb.filename = "zdrv.hdf5" From a1d31e75e69a9b2deaf4d453ff0fe95d248f583d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:16:27 +0000 Subject: [PATCH 163/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PopIII/popiii.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index d7fb04a7d..cf4a0b864 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -470,9 +470,9 @@ auto problem_main() -> int RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity - amrex::ParmParse const pp("popiii"); + amrex::ParmParse const pp("popiii"); Real densfloor = NAN; - pp.query("density_floor", densfloor); + pp.query("density_floor", densfloor); sim.densityFloor_ = densfloor; sim.tempFloor_ = 2.73 * (30.0 + 1.0); From d0fc050f535a8c859387f733776c0269dbeafe71 Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 16:02:41 +0100 Subject: [PATCH 164/165] use a different variable for parmparse --- src/PopIII/popiii.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index cf4a0b864..ba7443dee 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -470,9 +470,9 @@ auto problem_main() -> int RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity - amrex::ParmParse const pp("popiii"); + amrex::ParmParse const ppp("popiii"); Real densfloor = NAN; - pp.query("density_floor", densfloor); + ppp.query("density_floor", densfloor); sim.densityFloor_ = densfloor; sim.tempFloor_ = 2.73 * (30.0 + 1.0); From 55d18ff484e966ec99076654ce741fca7bd4eede Mon Sep 17 00:00:00 2001 From: Piyush Sharda Date: Mon, 5 Feb 2024 16:08:33 +0100 Subject: [PATCH 165/165] dont need to define density_floor, its defined in simulation.hpp --- src/PopIII/popiii.cpp | 5 ----- tests/PopIII.in | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/PopIII/popiii.cpp b/src/PopIII/popiii.cpp index ba7443dee..a4deb6ca2 100644 --- a/src/PopIII/popiii.cpp +++ b/src/PopIII/popiii.cpp @@ -470,11 +470,6 @@ auto problem_main() -> int RadhydroSimulation sim(BCs_cc); sim.doPoissonSolve_ = 1; // enable self-gravity - amrex::ParmParse const ppp("popiii"); - Real densfloor = NAN; - ppp.query("density_floor", densfloor); - sim.densityFloor_ = densfloor; - sim.tempFloor_ = 2.73 * (30.0 + 1.0); // sim.speedCeiling_ = 3e6; diff --git a/tests/PopIII.in b/tests/PopIII.in index c390b46ac..3810ada3b 100644 --- a/tests/PopIII.in +++ b/tests/PopIII.in @@ -42,7 +42,7 @@ jeansRefine.ncells = 64 #refine on these many cells per jeans length jeansRefine.jeans_density_threshold = 2e-20 #do not refine if density is less than this density # density floor for popiii -popiii.densfloor = 1e-25 +density_floor = 1e-25 # in quokka/src/StarCluster, generate with 'python3 perturbation.py --kmin=2 --kmax=64 --size=128 --alpha=2 --f_solenoidal=1.0' # and put it in quokka/tests/