From 291096a6b7960e6e122c4a142e48a37a305d3027 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Sun, 17 Nov 2024 17:54:45 -0800 Subject: [PATCH 01/13] Minor fix in FatesPFTIndexSwapper.py. At least with my scipy configuration, the code would crash with scalars. The updates should be equivalent to the old code but do not cause errors. --- tools/FatesPFTIndexSwapper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/FatesPFTIndexSwapper.py b/tools/FatesPFTIndexSwapper.py index c63f8891b7..07f71920c9 100755 --- a/tools/FatesPFTIndexSwapper.py +++ b/tools/FatesPFTIndexSwapper.py @@ -203,8 +203,9 @@ def main(argv): # Copy over the input data # Tedious, but I have to permute through all combinations of dimension position if( pft_dim_len == 0 ): - out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions)) - out_var.assignValue(float(fp_in.variables.get(key).data)) + # Scalar: do not assume any dimensions. + out_var = fp_out.createVariable(key,'d',()) + out_var[()] = in_var[()] elif( (pft_dim_found==-1) & (prt_dim_found==-1) & (litt_dim_found==-1) & (hydro_dim_found==-1) & (landuse_dim_found==-1) ): out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions)) out_var[:] = in_var[:] @@ -283,7 +284,7 @@ def main(argv): fp_in.close() fp_out.close() - print('Cloneing complete!') + print('Cloning complete!') exit(0) From fd7922f234822c7fe1c21fdc65660e82b4f17767 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:18:02 -0800 Subject: [PATCH 02/13] Editing modify_fates_paramfile.py similarly to FatesPFTIndexSwapper.py for scalars. --- tools/modify_fates_paramfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/modify_fates_paramfile.py b/tools/modify_fates_paramfile.py index a2a31188b3..074dba9f3d 100755 --- a/tools/modify_fates_paramfile.py +++ b/tools/modify_fates_paramfile.py @@ -133,7 +133,7 @@ def main(): for i in range(var.shape[0]): var[i] = outputval[i] elif(ndim_file==0): - var.assignValue(outputval[0]) + var[()] = outputval[()] else: print("Unhandled dimension size in modify_fates_paramfile.py") From 7539b402ae06ef790560d0a81b303cc886a25101 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:32:12 -0800 Subject: [PATCH 03/13] Updating ncvarsort.py so it uses the same netcdf libraries as other python scripts. --- tools/ncvarsort.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 6583700ae3..eaab228c20 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -4,11 +4,19 @@ # --input or --fin: input filename. # --output or --fout: output filename. If missing, will assume its directly modifying the input file, and will prompt unless -O is specified -import netCDF4 as nc +#import netCDF4 as nc import sys import os import argparse +# Newer versions of scipy have dropped the netcdf module and +# netcdf functions are part of the io parent module +try: + from scipy import io as nc + +except ImportError: + from scipy.io import netcdf as nc + # program sorts the variables based on the provided list, and pulls them one at a time # from an existing file and adds them to a new file in the sorted order. # input/output based on code here: https://gist.github.com/guziy/8543562 From 7d30a3fb6c43f0bef6cb6a37b64e427d514dc677 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:37:44 -0800 Subject: [PATCH 04/13] Append extension nc to the temporary files. In some combinations of python/netcdf libraries, the lack of nc extension has caused the function to crash. --- tools/BatchPatchParams.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/BatchPatchParams.py b/tools/BatchPatchParams.py index cd3e934632..9245e6dbea 100755 --- a/tools/BatchPatchParams.py +++ b/tools/BatchPatchParams.py @@ -107,13 +107,17 @@ def main(): base_cdl = xmlroot.find('base_file').text new_cdl = xmlroot.find('new_file').text + # Append extension nc to temporary files + # (in some netcdf versions, the lack of extension causes failures) + ext_nc = ".nc" + # Convert the base cdl file into a temp nc binary - base_nc = os.popen('mktemp').read().rstrip('\n') + base_nc = os.popen('mktemp').read().rstrip('\n')+ext_nc gencmd = "ncgen -o "+base_nc+" "+base_cdl os.system(gencmd) # Generate a temp output file name - new_nc = os.popen('mktemp').read().rstrip('\n') + new_nc = os.popen('mktemp').read().rstrip('\n')+ext_nc os.system("ls "+base_nc) os.system("ls "+new_nc) @@ -190,7 +194,7 @@ def main(): fp_nc.close() # Sort the new file - newer_nc = os.popen('mktemp').read().rstrip('\n') + newer_nc = os.popen('mktemp').read().rstrip('\n')+ext_nc os.system("../tools/ncvarsort.py --fin "+new_nc+" --fout "+newer_nc+" --overwrite") From 5cb89ad14775b291450869401a3ef0d2db5aa2dc Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:47:42 -0800 Subject: [PATCH 05/13] Additional edits to ncvarsort.py to align with the scipy/io/netcdf library. --- tools/ncvarsort.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index eaab228c20..7f35293690 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -33,7 +33,7 @@ def main(): args = parser.parse_args() # # open the input dataset - dsin = nc.Dataset(args.fnamein, 'r') + dsin = nc.netcdf_file(args.fnamein, 'r') # # make empty lists to hold the variable names in. the first of these is a list of sub-lists, # one for each type of variable (based on dimensionality). @@ -106,7 +106,7 @@ def main(): else: raise ValueError('Output file already exists and overwrite flag not specified for filename: '+args.fnameout) # - dsout = nc.Dataset(args.fnameout, "w") + dsout = nc.netcdf_file(args.fnameout, "w") # #Copy dimensions for dname, the_dim in dsin.dimensions.items(): @@ -126,14 +126,20 @@ def main(): # as well as all metadata to the new file. for i in range(len(varnames_list_sorted)): v_name = varnames_list_sorted[i] - varin = dsin.variables[v_name] + varin = dsin.variables.get[v_name] outVar = dsout.createVariable(v_name, varin.datatype, varin.dimensions) + + n_dimensions = len(varin.dimensions) if args.debug: if (verbose): print(v_name) # outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()}) - outVar[:] = varin[:] + if ( n_dimensions == 0): + outVar[()] = varin[()] + else: + outVar[:] = varin[:] # + # copy global attributes dsout.setncatts({k: dsin.getncattr(k) for k in dsin.ncattrs()})# # From f2b44986fa34a34342dd1b6c826eecbd3b8e76b6 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:51:41 -0800 Subject: [PATCH 06/13] Force integer in createDimension in ncvarsort.py I copied this from FatesPFTIndexSwapper.py, because the code was giving an attribute error. --- tools/ncvarsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 7f35293690..562b82374c 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -112,7 +112,7 @@ def main(): for dname, the_dim in dsin.dimensions.items(): if args.debug: if (verbose): print(dname, the_dim.size) - dsout.createDimension(dname, the_dim.size ) + dsout.createDimension(dname, int(the_dim.size) ) # if (verbose): print() # From b417a78e8ce657826517dae1558efe0b3bff5060 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:54:56 -0800 Subject: [PATCH 07/13] Dropping attribute size from "the_dim" in ncvartsort.py Apparently this is different between scipy/io/netcdf and netCDF4 --- tools/ncvarsort.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 562b82374c..a2d96cab43 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -111,8 +111,8 @@ def main(): #Copy dimensions for dname, the_dim in dsin.dimensions.items(): if args.debug: - if (verbose): print(dname, the_dim.size) - dsout.createDimension(dname, int(the_dim.size) ) + if (verbose): print(dname, the_dim) + dsout.createDimension(dname, int(the_dim) ) # if (verbose): print() # From 612d4e010edb270895518885bb2e81a1d284f855 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 09:57:58 -0800 Subject: [PATCH 08/13] Bug fix when retrieving variables in ncvarsort.py. Fixing a bug I introduced when I replaced variable retrieval with variables.get. I had forgotten to replace square brackets with parentheses. --- tools/ncvarsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index a2d96cab43..eb01fe52f2 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -126,7 +126,7 @@ def main(): # as well as all metadata to the new file. for i in range(len(varnames_list_sorted)): v_name = varnames_list_sorted[i] - varin = dsin.variables.get[v_name] + varin = dsin.variables.get(v_name) outVar = dsout.createVariable(v_name, varin.datatype, varin.dimensions) n_dimensions = len(varin.dimensions) From 7bf2f7cd75414edb3662f83f01b07d1b3da1f431 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 10:02:52 -0800 Subject: [PATCH 09/13] Replacing datatype with typecode in ncvarsort.py --- tools/ncvarsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index eb01fe52f2..98fa74002f 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -127,7 +127,7 @@ def main(): for i in range(len(varnames_list_sorted)): v_name = varnames_list_sorted[i] varin = dsin.variables.get(v_name) - outVar = dsout.createVariable(v_name, varin.datatype, varin.dimensions) + outVar = dsout.createVariable(v_name, varin.typecode, varin.dimensions) n_dimensions = len(varin.dimensions) if args.debug: From f36f34a550242b8dc7c6b5c8aec7910aa0757fdf Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 10:08:24 -0800 Subject: [PATCH 10/13] More fixes in ncvarsort.py to be compatible to scipy.io.netcdf --- tools/ncvarsort.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 98fa74002f..4a4eac8496 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -126,15 +126,18 @@ def main(): # as well as all metadata to the new file. for i in range(len(varnames_list_sorted)): v_name = varnames_list_sorted[i] - varin = dsin.variables.get(v_name) - outVar = dsout.createVariable(v_name, varin.typecode, varin.dimensions) + v_type = dsin.variables[v_name].type + v_dims = dsin.variables[v_name].dimensions + varin = dsin.variables.get(v_name) - n_dimensions = len(varin.dimensions) + outVar = dsout.createVariable(v_name, v_type, v_dims) + + n_dims = len(v_dims) if args.debug: if (verbose): print(v_name) # outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()}) - if ( n_dimensions == 0): + if ( n_dims == 0): outVar[()] = varin[()] else: outVar[:] = varin[:] From 14a5cb44c5d82c0eac2920058a5c1a121e39b95a Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 10:09:46 -0800 Subject: [PATCH 11/13] Fix typo in my last update --- tools/ncvarsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 4a4eac8496..070cd70c8e 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -126,7 +126,7 @@ def main(): # as well as all metadata to the new file. for i in range(len(varnames_list_sorted)): v_name = varnames_list_sorted[i] - v_type = dsin.variables[v_name].type + v_type = dsin.variables[v_name].typecode v_dims = dsin.variables[v_name].dimensions varin = dsin.variables.get(v_name) From 9b2384c921ee80b66a1d452ba38abaaf5ef93470 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Mon, 18 Nov 2024 10:27:52 -0800 Subject: [PATCH 12/13] Multiple fixes to ncvarsort.py to be compatible with scipy.io.netcdf --- tools/ncvarsort.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 070cd70c8e..255475b910 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -126,25 +126,31 @@ def main(): # as well as all metadata to the new file. for i in range(len(varnames_list_sorted)): v_name = varnames_list_sorted[i] - v_type = dsin.variables[v_name].typecode - v_dims = dsin.variables[v_name].dimensions varin = dsin.variables.get(v_name) - + v_type = dsin.variables[v_name].typecode() + v_dims = varin.dimensions + print(" V_NAME = ",v_name,"; V_TYPE = ",v_type,"; V_DIMS = ",v_dims) outVar = dsout.createVariable(v_name, v_type, v_dims) n_dims = len(v_dims) if args.debug: if (verbose): print(v_name) # - outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()}) + + # Copy attributes + for v_attr in varin._attributes: + setattr(outVar,v_attr,getattr(varin,v_attr)) + if ( n_dims == 0): outVar[()] = varin[()] else: outVar[:] = varin[:] # - # copy global attributes - dsout.setncatts({k: dsin.getncattr(k) for k in dsin.ncattrs()})# + # copy global attributes + for g_attr in dsin._attributes: + setattr(dsout,g_attr,getattr(dsin,g_attr)) + # # close the output file dsin.close() From 6a44ef4c27e1060a14d302338240c3bde4ae75c2 Mon Sep 17 00:00:00 2001 From: Marcos Longo Date: Tue, 17 Dec 2024 13:59:45 -0300 Subject: [PATCH 13/13] Removing print statement added for debugging. --- tools/ncvarsort.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ncvarsort.py b/tools/ncvarsort.py index 255475b910..92c8c85fc9 100755 --- a/tools/ncvarsort.py +++ b/tools/ncvarsort.py @@ -129,7 +129,6 @@ def main(): varin = dsin.variables.get(v_name) v_type = dsin.variables[v_name].typecode() v_dims = varin.dimensions - print(" V_NAME = ",v_name,"; V_TYPE = ",v_type,"; V_DIMS = ",v_dims) outVar = dsout.createVariable(v_name, v_type, v_dims) n_dims = len(v_dims)