file_min_digits: 5->6 (#2385)

* file_min_digits: 5->6

100k+ step runs are quite common in WarpX. To simplify
post-processing scripts, increase the default to pad to 6 digits.

This might break some hand-written scripts that use `?????`
wild-cards over `*` wildcards in regex and thus need to be updated.
But it at the same time simplifies regexes for analysis of data
series and listing of such file series.

* Update Scripts to Handle New File Names

* Fix CI Test pml_x_psatd

* Fix CI Test divb_cleaning_3d

* Fix CI Tests with EB Support

* Fix CI Tests with RZ Support

* Fix CI Tests with PICMI Support

* Use glob in Analysis of Collision Tests

* Use rstrip in Analysis of Collision Tests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix CI Tests for Langmuir_multi_1d/psatd

* Fix CI Tests for LaserInjection_1d

* Checkpoint-Restart Tests: 5 Digits

`regtest.py` in `regression_testing` hard-codes 5 digits:
```diff
--- a/regtest.py
+++ b/regtest.py
@@ -734,7 +734,7 @@ def test_suite(argv):
                 shutil.move(test.diffDir, orig_diff_dir)

             # get the file number to restart from
-            restart_file = "%s_chk%5.5d" % (test.name, test.restartFileNum)
+            restart_file = "%s_chk%6.6d" % (test.name, test.restartFileNum)
```

* PICMI Restart Tests: `warpx_file_min_digits = 5`

Same as other CI restart tests: hard-coded value in regression
suite makes this necessary.

X-ref: AMReX-Codes/regression_testing#119

Co-authored-by: Edoardo Zoni <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Docs/source/usage/parameters.rst

@@ -1943,7 +1943,7 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
 * ``<diag_name>.file_prefix`` (`string`) optional (default `diags/<diag_name>`)
     Root for output file names. Supports sub-directories.
 
-* ``<diag_name>.file_min_digits`` (`int`) optional (default `5`)
+* ``<diag_name>.file_min_digits`` (`int`) optional (default `6`)
     The minimum number of digits used for the iteration number appended to the diagnostic file names.
 
 * ``<diag_name>.diag_lo`` (list `float`, 1 per dimension) optional (default `-infinity -infinity -infinity`)

Examples/Modules/ParticleBoundaryProcess/analysis_absorption.py

@@ -9,11 +9,11 @@
 # the problem domain yet.
 
 # all particles are still there
-ds40 = yt.load("particle_absorption_plt00040")
+ds40 = yt.load("particle_absorption_plt000040")
 np40 = ds40.index.particle_headers['electrons'].num_particles
 assert(np40 == 612)
 
 # all particles have been removed
-ds60 = yt.load("particle_absorption_plt00060")
+ds60 = yt.load("particle_absorption_plt000060")
 np60 = ds60.index.particle_headers['electrons'].num_particles
 assert(np60 == 0)

Examples/Modules/ParticleBoundaryProcess/analysis_reflection.py

@@ -9,7 +9,7 @@
 
 import yt
 
-plotfile = 'Python_particle_reflection_plt00010'
+plotfile = 'Python_particle_reflection_plt000010'
 ds = yt.load( plotfile )  # noqa
 
 assert True

Examples/Modules/ParticleBoundaryScrape/analysis_scrape.py

@@ -11,19 +11,19 @@
 # the problem domain yet.
 
 # all particles are still there
-if Path("particle_scrape_plt00040").is_dir():
-    filename = "particle_scrape_plt00040"
+if Path("particle_scrape_plt000040").is_dir():
+    filename = "particle_scrape_plt000040"
 else:
-    filename = "Python_particle_scrape_plt00040"
+    filename = "Python_particle_scrape_plt000040"
 ds40 = yt.load(filename)
 np40 = ds40.index.particle_headers['electrons'].num_particles
 assert(np40 == 612)
 
 # all particles have been removed
-if Path("particle_scrape_plt00060").is_dir():
-    filename = "particle_scrape_plt00060"
+if Path("particle_scrape_plt000060").is_dir():
+    filename = "particle_scrape_plt000060"
 else:
-    filename = "Python_particle_scrape_plt00060"
+    filename = "Python_particle_scrape_plt000060"
 ds60 = yt.load(filename)
 np60 = ds60.index.particle_headers['electrons'].num_particles
 assert(np60 == 0)

Examples/Modules/laser_injection/analysis_1d.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2021 Prabhat Kumar, Remi Lehe
+# Copyright 2021-2022 Prabhat Kumar, Remi Lehe, Axel Huebl
 #
 # This file is part of WarpX.
 #
@@ -12,6 +12,7 @@
 # the simulation and it compares it with theory. It also checks that the
 # central frequency of the Fourier transform is the expected one.
 
+import os
 import sys
 
 import matplotlib
@@ -171,7 +172,7 @@ def main():
 
     check_laser(filename_end)
 
-    test_name = filename_end[:-9] # Could also be os.path.split(os.getcwd())[1]
+    test_name = os.path.split(os.getcwd())[1]
     checksumAPI.evaluate_checksum(test_name, filename_end)
 
 if __name__ == "__main__":

Examples/Modules/laser_injection_from_file/analysis.py

@@ -211,10 +211,10 @@ def do_analysis(fname, compname, steps):
 def launch_analysis(executable):
     create_gaussian_2d()
     os.system("./" + executable + " inputs.2d_test_txye diag1.file_prefix=diags/plotfiles/plt")
-    do_analysis("diags/plotfiles/plt00250/", "comp_unf.pdf", 250)
+    do_analysis("diags/plotfiles/plt000250/", "comp_unf.pdf", 250)
     os.system("sed 's/gauss_2d_unf.txye/gauss_2d.txye/g' inputs.2d_test_txye > inputs.2d_test_txye_non_unf")
     os.system("./" + executable + " inputs.2d_test_txye_non_unf diag1.file_prefix=diags/plotfiles/plt")
-    do_analysis("diags/plotfiles/plt00250/", "comp_non_unf.pdf", 250)
+    do_analysis("diags/plotfiles/plt000250/", "comp_non_unf.pdf", 250)
 
 
 def main() :

Examples/Physics_applications/capacitive_discharge/analysis.py

@@ -14,6 +14,6 @@
 import checksumAPI
 
 my_check = checksumAPI.evaluate_checksum(
-    'background_mcc', 'Python_background_mcc_plt00050',
+    'background_mcc', 'Python_background_mcc_plt000050',
     do_particles=True, rtol=3.7e-3
 )

Examples/Tests/ElectrostaticSphereEB/analysis.py

@@ -11,6 +11,6 @@
 import checksumAPI
 
 my_check = checksumAPI.evaluate_checksum(
-    'ElectrostaticSphereEB', 'Python_ElectrostaticSphereEB_plt00001',
+    'ElectrostaticSphereEB', 'Python_ElectrostaticSphereEB_plt000001',
     do_particles=False, atol=1e-12
 )

Examples/Tests/Langmuir/analysis_langmuir_multi.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2019 Jean-Luc Vay, Maxence Thevenet, Remi Lehe
+# Copyright 2019-2022 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Axel Huebl
 #
 #
 # This file is part of WarpX.
@@ -13,6 +13,7 @@
 # $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$
 # $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$
 # $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$
+import os
 import re
 import sys
 
@@ -23,7 +24,6 @@
 import yt
 
 yt.funcs.mylog.setLevel(50)
-import os
 
 import numpy as np
 from scipy.constants import c, e, epsilon_0, m_e
@@ -151,8 +151,8 @@ def get_theoretical_field( field, t ):
     assert( error_rel < tolerance )
 
 if div_cleaning:
-    ds_old = yt.load('Langmuir_multi_psatd_div_cleaning_plt00038')
-    ds_mid = yt.load('Langmuir_multi_psatd_div_cleaning_plt00039')
+    ds_old = yt.load('Langmuir_multi_psatd_div_cleaning_plt000038')
+    ds_mid = yt.load('Langmuir_multi_psatd_div_cleaning_plt000039')
     ds_new = yt.load(fn) # this is the last plotfile
 
     ad_old = ds_old.covering_grid(level = 0, left_edge = ds_old.domain_left_edge, dims = ds_old.domain_dimensions)

Examples/Tests/Langmuir/analysis_langmuir_multi_1d.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2019-2021 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Prabhat Kumar
+# Copyright 2019-2022 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Prabhat Kumar, Axel Huebl
 #
 #
 # This file is part of WarpX.
@@ -11,6 +11,7 @@
 # the script `inputs.multi.rt`. This simulates a 1D periodic plasma wave.
 # The electric field in the simulation is given (in theory) by:
 # $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\sin(k_z z)\sin( \omega_p t)$$
+import os
 import re
 import sys
 
@@ -21,6 +22,7 @@
 import yt
 
 yt.funcs.mylog.setLevel(50)
+
 import numpy as np
 from scipy.constants import c, e, epsilon_0, m_e
 
@@ -75,7 +77,7 @@ def get_theoretical_field( field, t ):
 # Check the validity of the fields
 error_rel = 0
 for field in ['Ez']:
-    E_sim = data[field].to_ndarray()[:,0,0]
+    E_sim = data[('mesh',field)].to_ndarray()[:,0,0]
     E_th = get_theoretical_field(field, t0)
     max_error = abs(E_sim-E_th).max()/abs(E_th).max()
     print('%s: Max error: %.2e' %(field,max_error))
@@ -104,14 +106,14 @@ def get_theoretical_field( field, t ):
 # current correction (psatd.do_current_correction=1) is applied or when
 # Vay current deposition (algo.current_deposition=vay) is used
 if current_correction or vay_deposition:
-    rho  = data['rho' ].to_ndarray()
-    divE = data['divE'].to_ndarray()
+    rho  = data[('boxlib','rho')].to_ndarray()
+    divE = data[('boxlib','divE')].to_ndarray()
     error_rel = np.amax( np.abs( divE - rho/epsilon_0 ) ) / np.amax( np.abs( rho/epsilon_0 ) )
     tolerance = 1.e-9
     print("Check charge conservation:")
     print("error_rel = {}".format(error_rel))
     print("tolerance = {}".format(tolerance))
     assert( error_rel < tolerance )
 
-test_name = fn[:-9] # Could also be os.path.split(os.getcwd())[1]
+test_name = os.path.split(os.getcwd())[1]
 checksumAPI.evaluate_checksum(test_name, fn)

Examples/Tests/Langmuir/analysis_langmuir_multi_rz.py

@@ -140,17 +140,17 @@ def Ez( z, r, epsilon, k0, w0, wp, t) :
 dim = "rz"
 species_name = "electrons"
 
-parser_filter_fn = "diags/diag_parser_filter00080"
+parser_filter_fn = "diags/diag_parser_filter000080"
 parser_filter_expression = "(py-pz < 0) * (r<10e-6) * (z > 0)"
 post_processing_utils.check_particle_filter(fn, parser_filter_fn, parser_filter_expression,
                                             dim, species_name)
 
-uniform_filter_fn = "diags/diag_uniform_filter00080"
+uniform_filter_fn = "diags/diag_uniform_filter000080"
 uniform_filter_expression = "ids%3 == 0"
 post_processing_utils.check_particle_filter(fn, uniform_filter_fn, uniform_filter_expression,
                                             dim, species_name)
 
-random_filter_fn = "diags/diag_random_filter00080"
+random_filter_fn = "diags/diag_random_filter000080"
 random_fraction = 0.66
 post_processing_utils.check_random_filter(fn, random_filter_fn, random_fraction,
                                           dim, species_name)

Examples/Tests/PML/analysis_pml_psatd.py

@@ -25,7 +25,7 @@
 energy_start = 7.282940107273505e-08 # electromagnetic energy at iteration 50
 
 # Check consistency of field energy diagnostics with initial energy above
-ds = yt.load('pml_x_psatd_plt00050')
+ds = yt.load('pml_x_psatd_plt000050')
 all_data_level_0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions)
 Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze()
 By = all_data_level_0['boxlib', 'By'].v.squeeze()

Examples/Tests/collision/analysis_collision_2d.py

@@ -58,7 +58,7 @@
 # Remove trailing '/' from file name, if necessary
 last_fn.rstrip('/')
 # Find last iteration in file name, such as 'test_name_plt000001' (last_it = '000001')
-last_it = re.search('\d+$', last_fn).group()
+last_it = re.search('\d+', last_fn).group()
 # Find output prefix in file name, such as 'test_name_plt000001' (prefix = 'test_name_plt')
 prefix = last_fn[:-len(last_it)]
 # Collect all output files in fn_list (names match pattern prefix + arbitrary number)

Examples/Tests/collision/analysis_collision_3d.py

@@ -58,7 +58,7 @@
 # Remove trailing '/' from file name, if necessary
 last_fn.rstrip('/')
 # Find last iteration in file name, such as 'test_name_plt000001' (last_it = '000001')
-last_it = re.search('\d+$', last_fn).group()
+last_it = re.search('\d+', last_fn).group()
 # Find output prefix in file name, such as 'test_name_plt000001' (prefix = 'test_name_plt')
 prefix = last_fn[:-len(last_it)]
 # Collect all output files in fn_list (names match pattern prefix + arbitrary number)

Examples/Tests/divb_cleaning/analysis.py

@@ -24,8 +24,8 @@
 fn = sys.argv[1]
 
 # Load yt data
-ds_old = yt.load('divb_cleaning_3d_plt00398')
-ds_mid = yt.load('divb_cleaning_3d_plt00399')
+ds_old = yt.load('divb_cleaning_3d_plt000398')
+ds_mid = yt.load('divb_cleaning_3d_plt000399')
 ds_new = yt.load(fn) # this is the last plotfile
 
 ad_old = ds_old.covering_grid(level = 0, left_edge = ds_old.domain_left_edge, dims = ds_old.domain_dimensions)

Examples/Tests/pass_mpi_communicator/analysis.py

@@ -17,12 +17,12 @@
 import checksum
 
 # this will be the name of the first plot file
-fn1 = "Python_pass_mpi_comm_plt1_00010"
+fn1 = "Python_pass_mpi_comm_plt1_000010"
 # second plot file
-fn2 = "Python_pass_mpi_comm_plt2_00010"
+fn2 = "Python_pass_mpi_comm_plt2_000010"
 
-test_name1 = fn1[:-9]
-test_name2 = fn2[:-9]
+test_name1 = fn1[:-10]
+test_name2 = fn2[:-10]
 
 
 checksum1 = checksum.Checksum(test_name1, fn1, do_fields=True,

Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py

@@ -75,6 +75,7 @@
     name = 'chkpoint',
     period = 5,
     write_dir = '.',
+    warpx_file_min_digits = 5,
     warpx_file_prefix = f'Python_restart_runtime_components_chk'
 )
 

Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py

@@ -86,6 +86,7 @@
     name = 'chkpoint',
     period = diagnostic_intervals,
     write_dir = '.',
+    warpx_file_min_digits = 5,
     warpx_file_prefix = f'Python_restart_eb_chk'
 )
 

Regression/WarpX-tests.ini

@@ -71,7 +71,7 @@ cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON
 [pml_x_yee]
 buildDir = .
 inputFile = Examples/Tests/PML/inputs_2d
-runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_chk
+runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_chk chk.file_min_digits=5
 dim = 2
 addToCompileString =
 cmakeSetupOpts = -DWarpX_DIMS=2
@@ -104,7 +104,7 @@ analysisRoutine = Examples/Tests/PML/analysis_pml_ckc.py
 [pml_x_psatd]
 buildDir = .
 inputFile = Examples/Tests/PML/inputs_2d
-runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk
+runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5
 dim = 2
 addToCompileString = USE_PSATD=TRUE
 cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON
@@ -1190,7 +1190,7 @@ analysisRoutine = Examples/analysis_default_regression.py
 [uniform_plasma_restart]
 buildDir = .
 inputFile = Examples/Physics_applications/uniform_plasma/inputs_3d
-runtime_params = chk.file_prefix=uniform_plasma_restart_chk
+runtime_params = chk.file_prefix=uniform_plasma_restart_chk chk.file_min_digits=5
 dim = 3
 addToCompileString =
 cmakeSetupOpts = -DWarpX_DIMS=3
@@ -1209,7 +1209,7 @@ analysisRoutine = Examples/Tests/restart/analysis_restart.py
 [restart]
 buildDir = .
 inputFile = Examples/Tests/restart/inputs
-runtime_params = chk.file_prefix=restart_chk
+runtime_params = chk.file_prefix=restart_chk chk.file_min_digits=5
 dim = 3
 addToCompileString =
 cmakeSetupOpts = -DWarpX_DIMS=3
@@ -1228,7 +1228,7 @@ analysisRoutine = Examples/Tests/restart/analysis_restart.py
 [restart_psatd]
 buildDir = .
 inputFile = Examples/Tests/restart/inputs
-runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped
+runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped
 dim = 3
 addToCompileString = USE_PSATD=TRUE
 cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON
@@ -1247,7 +1247,7 @@ analysisRoutine = Examples/Tests/restart/analysis_restart.py
 [restart_psatd_time_avg]
 buildDir = .
 inputFile = Examples/Tests/restart/inputs
-runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_time_avg_chk boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.do_time_averaging=1
+runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_time_avg_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.do_time_averaging=1
 dim = 3
 addToCompileString = USE_PSATD=TRUE
 cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON

Source/Diagnostics/Diagnostics.H

@@ -130,7 +130,7 @@ protected:
     /** Prefix for output directories */
     std::string m_file_prefix;
     /** Minimum number of digits to iteration number in file name */
-    int m_file_min_digits = 5;
+    int m_file_min_digits = 6;
     /** Index of diagnostics in MultiDiagnostics::alldiags */
     int m_diag_index;
     /** Names of  each component requested by the user.