From f3951fffe3cbf0efc657866c3895283c78b08880 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Mon, 29 Jul 2024 12:29:34 +1000 Subject: [PATCH 01/27] Add grid checking functions --- umpost/um2netcdf.py | 157 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 32 deletions(-) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 31e3f53..459cc90 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -64,8 +64,10 @@ def pg_calendar(self): # TODO: rename time to avoid clash with builtin time module def convert_proleptic(time): # Convert units from hours to days and shift origin from 1970 to 0001 - newunits = cf_units.Unit("days since 0001-01-01 00:00", calendar='proleptic_gregorian') - tvals = np.array(time.points) # Need a copy because can't assign to time.points[i] + newunits = cf_units.Unit( + "days since 0001-01-01 00:00", calendar='proleptic_gregorian') + # Need a copy because can't assign to time.points[i] + tvals = np.array(time.points) tbnds = np.array(time.bounds) if time.bounds is not None else None for i in range(len(time.points)): @@ -89,6 +91,103 @@ def convert_proleptic(time): time.units = newunits +def fix_latlon_coord_names(cube, grid_type, dlat, dlon): + """ + Rename cube's latitude/longitude coordinate variables based on + the grid they lie on. + + Parameters + ---------- + cube: an Iris cube object to modify (changes names in place). + grid_type: (string) model horizontal grid type. + dlat: (float) meridional spacing between latitude grid points. + NB - Only applies to variables on the main horizontal grids, + and not the river grid. + dlon: (float) zonal spacing between longitude grid points. + NB - Only applies to variables on the main horizontal grids, + and not the river grid. + """ + + lat = cube.coord('latitude') + if is_lat_river(lat.points): + lat.var_name = 'lat_river' + elif is_lat_v(lat.points, grid_type, dlat): + lat.var_name = 'lat_v' + else: + lat.var_name = 'lat' + + lon = cube.coord('longitude') + if is_lon_river(lon.points): + lon.var_name = 'lon_river' + elif is_lon_u(lon.points, grid_type, dlon): + lon.var_name = 'lon_u' + else: + lon.var_name = 'lon' + + +def is_lat_river(latitude_points): + """ + Check whether latitude points are on the river routing grid. + + Parameters + ---------- + latitude_points: (array) 1D array of latitude grid points. + """ + return len(latitude_points) == 180 + + +def is_lon_river(longitude_points): + """ + Check whether longitude points are on the river routing grid. + + Parameters + ---------- + longitude_points: (array) 1D array of longitude grid points. + """ + + return len(longitude_points) == 360 + + +def is_lat_v(latitude_points, grid_type, dlat): + """ + Check whether latitude points are on the lat_v grid. + + Parameters + ---------- + latitude_points: (array) 1D array of latitude grid points. + grid_type: (string) model horizontal grid type. + dlat: (float) meridional spacing between latitude grid points. + """ + + return ( + (latitude_points[0] == -90 and grid_type == GRID_END_GAME) or + ( + np.allclose(-90.+0.5*dlat, latitude_points[0]) and + grid_type == GRID_NEW_DYNAMICS + ) + ) + + +def is_lon_u(longitude_points, grid_type, dlon): + """ + Check whether longitude points are on the lon_u grid. + + Parameters + ---------- + longitude: (array) 1D array of longitude grid points. + grid_type: (string) model horizontal grid type. + dlon: (float) zonal spacing between longitude grid points. + """ + + return ( + (longitude_points[0] == 0 and grid_type == GRID_END_GAME) or + ( + np.allclose(0.5*dlon, longitude_points[0]) and + grid_type == GRID_NEW_DYNAMICS + ) + ) + + def fix_latlon_coord(cube, grid_type, dlat, dlon): def _add_coord_bounds(coord): if len(coord.points) > 1: @@ -112,24 +211,6 @@ def _add_coord_bounds(coord): lon.points = lon.points.astype(np.float64) _add_coord_bounds(lon) - lat = cube.coord('latitude') - if len(lat.points) == 180: - lat.var_name = 'lat_river' - elif (lat.points[0] == -90 and grid_type == 'EG') or \ - (np.allclose(-90.+0.5*dlat, lat.points[0]) and grid_type == 'ND'): - lat.var_name = 'lat_v' - else: - lat.var_name = 'lat' - - lon = cube.coord('longitude') - if len(lon.points) == 360: - lon.var_name = 'lon_river' - elif (lon.points[0] == 0 and grid_type == 'EG') or \ - (np.allclose(0.5*dlon, lon.points[0]) and grid_type == 'ND'): - lon.var_name = 'lon_u' - else: - lon.var_name = 'lon' - # TODO: refactor to "rename level coord" def fix_level_coord(cube, z_rho, z_theta): @@ -182,7 +263,8 @@ def cubewrite(cube, sman, compression, use64bit, verbose): fill_value = 1.e20 else: # Use netCDF defaults - fill_value = default_fillvals['%s%1d' % (cube.data.dtype.kind, cube.data.dtype.itemsize)] + fill_value = default_fillvals['%s%1d' % ( + cube.data.dtype.kind, cube.data.dtype.itemsize)] cube.attributes['missing_value'] = np.array([fill_value], cube.data.dtype) @@ -202,7 +284,8 @@ def cubewrite(cube, sman, compression, use64bit, verbose): else: new_calendar = time.units.calendar - time.units = cf_units.Unit("days since 1970-01-01 00:00", calendar=new_calendar) + time.units = cf_units.Unit( + "days since 1970-01-01 00:00", calendar=new_calendar) time.points = time.points/24. if time.bounds is not None: @@ -251,7 +334,8 @@ def cubewrite(cube, sman, compression, use64bit, verbose): except iris.exceptions.CoordinateNotFoundError: # No time dimension (probably ancillary file) - sman.write(cube, zlib=True, complevel=compression, fill_value=fill_value) + sman.write(cube, zlib=True, complevel=compression, + fill_value=fill_value) def fix_cell_methods(mtuple): @@ -276,7 +360,8 @@ def apply_mask(c, heaviside, hcrit): # Temporarily turn off warnings from 0/0 # TODO: refactor to use np.where() with np.errstate(divide='ignore', invalid='ignore'): - c.data = np.ma.masked_array(c.data/heaviside.data, heaviside.data <= hcrit).astype(np.float32) + c.data = np.ma.masked_array( + c.data/heaviside.data, heaviside.data <= hcrit).astype(np.float32) else: # Are the levels of c a subset of the levels of the heaviside variable? c_p = c.coord('pressure') @@ -288,11 +373,14 @@ def apply_mask(c, heaviside, hcrit): h_tmp = heaviside.extract(constraint) # Double check they're actually the same after extraction if not np.all(c_p.points == h_tmp.coord('pressure').points): - raise Exception('Unexpected mismatch in levels of extracted heaviside function') + raise Exception( + 'Unexpected mismatch in levels of extracted heaviside function') with np.errstate(divide='ignore', invalid='ignore'): - c.data = np.ma.masked_array(c.data/h_tmp.data, h_tmp.data <= hcrit).astype(np.float32) + c.data = np.ma.masked_array( + c.data/h_tmp.data, h_tmp.data <= hcrit).astype(np.float32) else: - raise Exception('Unable to match levels of heaviside function to variable %s' % c.name()) + raise Exception( + 'Unable to match levels of heaviside function to variable %s' % c.name()) def process(infile, outfile, args): @@ -301,7 +389,8 @@ def process(infile, outfile, args): ff = mule.load_umfile(str(infile)) if isinstance(ff, mule.ancil.AncilFile): - raise NotImplementedError('Ancillary files are currently not supported') + raise NotImplementedError( + 'Ancillary files are currently not supported') # TODO: eventually move these calls closer to their usage grid_type = get_grid_type(ff) @@ -341,7 +430,8 @@ def process(infile, outfile, args): umvar = stashvar.StashVar(c.item_code) if args.simple: - c.var_name = 'fld_s%2.2di%3.3d' % (stashcode.section, stashcode.item) + c.var_name = 'fld_s%2.2di%3.3d' % ( + stashcode.section, stashcode.item) elif umvar.uniquename: c.var_name = umvar.uniquename @@ -398,7 +488,8 @@ def process(infile, outfile, args): try: fix_latlon_coord(c, grid_type, dlat, dlon) except iris.exceptions.CoordinateNotFoundError: - print('\nMissing lat/lon coordinates for variable (possible timeseries?)\n') + print( + '\nMissing lat/lon coordinates for variable (possible timeseries?)\n') print(c) raise Exception("Variable can not be processed") @@ -443,7 +534,8 @@ def get_grid_type(ff): elif staggering == 3: return GRID_NEW_DYNAMICS else: - raise PostProcessingError(f"Unable to determine grid staggering from header '{staggering}'") + raise PostProcessingError( + f"Unable to determine grid staggering from header '{staggering}'") def get_grid_spacing(ff): @@ -635,7 +727,8 @@ def add_global_history(infile, iris_out): if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Convert UM fieldsfile to netcdf") + parser = argparse.ArgumentParser( + description="Convert UM fieldsfile to netcdf") parser.add_argument('-k', dest='nckind', required=False, type=int, default=3, help=('specify netCDF output format: 1 classic, 2 64-bit' From 088378c6de2a8a181e8ab1f5b576320132c81ddd Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Wed, 31 Jul 2024 16:21:20 +1000 Subject: [PATCH 02/27] continue refactor and add tests --- test/test_um2netcdf.py | 191 +++++++++++++++++++++++++++++++++++++++++ umpost/um2netcdf.py | 140 +++++++++++++++++++++++------- 2 files changed, 299 insertions(+), 32 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index df1781f..49ff8a0 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -1,6 +1,7 @@ import unittest.mock as mock from dataclasses import dataclass from collections import namedtuple +import numpy as np import umpost.um2netcdf as um2nc @@ -409,3 +410,193 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): for unit in ("", None): um2nc.fix_units(ua_plev_cube, unit, verbose=False) assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units + + +def test_is_lat_river(): + lat_river_points = np.arange(-90., 90) + 0.5 + assert len(lat_river_points) == 180 + assert um2nc.is_lat_river(lat_river_points) + + # latitude points on ESM1.5 N96 + not_lat_river_points = np.arange(-88.75, 89., 1.25) + not_lat_river_points = np.concatenate(([-90], + not_lat_river_points, + [90.] + )) + assert len(not_lat_river_points) != 180 + assert not um2nc.is_lat_river(not_lat_river_points) + + +def test_is_lon_river(): + lon_river_points = np.arange(0., 360.) + 0.5 + assert len(lon_river_points) == 360 + assert um2nc.is_lon_river(lon_river_points) + + # longitude points on normal ESM1.5 N96 grid + not_lon_river_points = np.arange(0., 360., 1.875) + assert len(not_lon_river_points) != 360 + assert not um2nc.is_lon_river(not_lon_river_points) + + +def test_is_lat_v_EG(): + dlat = 1.25 + lat_v_points = np.arange(-90., 90, dlat) + assert lat_v_points[0] == -90 + assert um2nc.is_lat_v(lat_v_points, um2nc.GRID_END_GAME, dlat) + + not_lat_v_points = np.arange(-88.5, 90, dlat) + assert not_lat_v_points[0] != -90 + assert not um2nc.is_lat_v(not_lat_v_points, um2nc.GRID_END_GAME, dlat) + + +def test_is_lat_v_NG(): + dlat = 1.25 + lat_v_points = np.arange(-90.+0.5*dlat, 90, dlat) + assert lat_v_points[0] == -90.+0.5*dlat + assert um2nc.is_lat_v(lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) + + not_lat_v_points = np.arange(-90., 90, dlat) + assert not um2nc.is_lat_v(not_lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) + + +def test_is_lon_u_EG(): + dlon = 1.875 + lon_u_points = np.arange(0, 360, dlon) + assert lon_u_points[0] == 0 + assert um2nc.is_lon_u(lon_u_points, um2nc.GRID_END_GAME, dlon) + + not_lon_u_points = np.arange(0.5, 360, dlon) + assert not_lon_u_points[0] != 0 + assert not um2nc.is_lon_u(not_lon_u_points, um2nc.GRID_END_GAME, dlon) + + +def test_is_lon_u_ND(): + dlon = 1.875 + lon_u_points = np.arange(0.5*dlon, 360, dlon) + assert lon_u_points[0] == 0.5*dlon + assert um2nc.is_lon_u(lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + + not_lon_u_points = np.arange(0, 360, dlon) + assert not_lon_u_points[0] != 0.5*dlon + assert not um2nc.is_lon_u(not_lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + + +@dataclass +class DummyCoordinate: + """ + Imitation cube coordinate for unit testing. + """ + coordname: str + points: np.ndarray + bounds: np.ndarray = None + + def name(self): + return self.coordname + def has_bounds(self): + return self.bounds is not None + + +def test_add_latlon_coord_bounds_has_bounds(): + # Test that bounds are not modified if they already exist + lon_points = np.array([1., 2., 3.]) + lon_bounds = np.array([[0.5, 1.5], + [1.5, 2.5], + [2.5, 3.5]]) + lon_coord_with_bounds = DummyCoordinate( + um2nc.LON_COORD_NAME, + lon_points, + lon_bounds + ) + assert lon_coord_with_bounds.has_bounds() + + um2nc.add_latlon_coord_bounds(lon_coord_with_bounds) + assert np.array_equal(lon_coord_with_bounds.bounds, lon_bounds) + + +def test_add_latlon_coord_guess_bounds(): + # Test that guess_bounds method is called when + # coordinate has no bounds and length > 1. + lon_points = np.array([0., 1.]) + lon_coord_nobounds = DummyCoordinate( + um2nc.LON_COORD_NAME, + lon_points + ) + + # Mock Iris' guess_bounds method to check whether it is called + lon_coord_nobounds.guess_bounds = mock.Mock(return_value=None) + + assert len(lon_coord_nobounds.points) > 1 + assert not lon_coord_nobounds.has_bounds() + + um2nc.add_latlon_coord_bounds(lon_coord_nobounds) + + lon_coord_nobounds.guess_bounds.assert_called() + + +def test_add_latlon_coord_single(): + for coord_name in [um2nc.LON_COORD_NAME, um2nc.LAT_COORD_NAME]: + points = np.array([0.]) + coord_single_point = DummyCoordinate( + coord_name, + points + ) + + assert len(coord_single_point.points) == 1 + assert not coord_single_point.has_bounds() + + um2nc.add_latlon_coord_bounds(coord_single_point) + + expected_bounds = um2nc.GLOBAL_COORD_BOUNDS[coord_name] + assert np.array_equal(coord_single_point.bounds, expected_bounds) + + +def test_add_latlon_coord_error(): + fake_coord_name = "fake coordinate" + fake_points = np.array([1., 2., 3.]) + + fake_coord = DummyCoordinate( + fake_coord_name, + fake_points + ) + + with pytest.raises(ValueError): + um2nc.add_latlon_coord_bounds(fake_coord) + + +class DummyCubeWithCoords(DummyCube): + # Dummy cube with coordinates, which can be filled with + # DummyCoordinate objects for testing. + def __init__(self, item_code, var_name=None, attributes=None, units=None, coords = {}): + super().__init__(item_code, var_name, attributes, units) + self.coordinate_dict = coords + + def coord(self, coordinate_name): + return self.coordinate_dict[coordinate_name] + + + +@pytest.fixture +def cube_with_latlon_coords(): + lat_points = np.array([-90., -88.75, -87.5 ], dtype = "float32") + lon_points = np.array([ 0., 1.875, 3.75 ], dtype = "float32") + + lat_coord_object = DummyCoordinate( + um2nc.LAT_COORD_NAME, + lat_points + ) + lon_coord_object = DummyCoordinate( + um2nc.LON_COORD_NAME, + lon_points + ) + + coords_dict = { + [um2nc.LAT_COORD_NAME]: lat_coord_object, + [um2nc.LON_COORD_NAME]: lon_coord_object + } + + + + +# TODO test_fix_latlon_coord_names(): + +# TODO test_fix_latlon_coords(): \ No newline at end of file diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 23ffc3f..bf10f75 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -40,6 +40,16 @@ # TODO: what is this limit & does it still exist? XCONV_LONG_NAME_LIMIT = 110 +LON_COORD_NAME = "longitude" +LAT_COORD_NAME = "latitude" + +# Bounds for global single cells +GLOBAL_COORD_BOUNDS = { + LON_COORD_NAME: np.array([[0., 360.]]), + LAT_COORD_NAME: np.array([[-90., 90.]]) +} + + class PostProcessingError(Exception): """Generic class for um2nc specific errors.""" @@ -161,11 +171,12 @@ def is_lat_v(latitude_points, grid_type, dlat): grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. """ - + + first_latitude = latitude_points[0] return ( - (latitude_points[0] == -90 and grid_type == GRID_END_GAME) or + (first_latitude == -90 and grid_type == GRID_END_GAME) or ( - np.allclose(-90.+0.5*dlat, latitude_points[0]) and + np.allclose(-90.+0.5*dlat, first_latitude) and grid_type == GRID_NEW_DYNAMICS ) ) @@ -181,38 +192,109 @@ def is_lon_u(longitude_points, grid_type, dlon): grid_type: (string) model horizontal grid type. dlon: (float) zonal spacing between longitude grid points. """ + first_longitude = longitude_points[0] + + ND_first_lon_u = 0.5*dlon return ( - (longitude_points[0] == 0 and grid_type == GRID_END_GAME) or + (first_longitude == 0 and grid_type == GRID_END_GAME) or ( - np.allclose(0.5*dlon, longitude_points[0]) and + np.allclose(ND_first_lon_u, first_longitude) and grid_type == GRID_NEW_DYNAMICS ) ) -def fix_latlon_coord(cube, grid_type, dlat, dlon): - def _add_coord_bounds(coord): - if len(coord.points) > 1: - if not coord.has_bounds(): - coord.guess_bounds() - else: - # For length 1, assume it's global. guess_bounds doesn't work in this case - if coord.name() == 'latitude': - if not coord.has_bounds(): - coord.bounds = np.array([[-90., 90.]]) - elif coord.name() == 'longitude': - if not coord.has_bounds(): - coord.bounds = np.array([[0., 360.]]) - lat = cube.coord('latitude') +def add_latlon_coord_bounds(cube_coordinate): + """ + Add bounds to horizontal coordinate (longitude or latitude) if + they don't already exist. Edits coordinate in place. - # Force to double for consistency with CMOR - lat.points = lat.points.astype(np.float64) - _add_coord_bounds(lat) - lon = cube.coord('longitude') - lon.points = lon.points.astype(np.float64) - _add_coord_bounds(lon) + Parameters + ---------- + cube_coordinate: coordinate object from iris cube. + """ + coordinate_name = cube_coordinate.name() + if coordinate_name not in [LON_COORD_NAME, LAT_COORD_NAME]: + raise ValueError( + f"Wrong coordinate {coordinate_name} supplied. " + f"Expected one of {LON_COORD_NAME}, {LAT_COORD_NAME}." + ) + + if cube_coordinate.has_bounds(): + # Don't change bounds if already exist + return + + if len(cube_coordinate.points) > 1: + cube_coordinate.guess_bounds() + else: + # For length 1, assume it's global. guess_bounds doesn't work in this case + cube_coordinate.bounds = GLOBAL_COORD_BOUNDS[coordinate_name] + + + +def fix_latlon_coords(cube, grid_type, dlat, dlon): + """ + Wrapper function to modify cube's horizontal coordinates + (latitude and longitude). Converts to float64, adds grid bounds, + and renames coordinates. Modifies cube in place. + + Parameters + ---------- + cube: Iris cube object (modified in place). + grid_type: (string) model horizontal grid type. + dlat: (float) meridional spacing between latitude grid points. + NB - Only applies to variables on the main horizontal grids, + and not the river grid. + dlon: (float) zonal spacing between longitude grid points. + NB - Only applies to variables on the main horizontal grids, + and not the river grid. + """ + + # TODO: Check that we don't need the double coordinate lookup. + try: + latitude_coordinate = cube.coord(LAT_COORD_NAME) + longitude_coordinate = cube.coord(LON_COORD_NAME) + # Force to double for consistency with CMOR + latitude_coordinate.points = latitude_coordinate.points.astype(np.float64) + longitude_coordinate.points = longitude_coordinate.points.astype(np.float64) + + add_latlon_coord_bounds(latitude_coordinate) + add_latlon_coord_bounds(longitude_coordinate) + + # Coordinate names should only be changed after the bounds are added. + fix_latlon_coord_names(cube, grid_type, dlat, dlon) + + except iris.exceptions.CoordinateNotFoundError: + print( + '\nMissing lat/lon coordinates for variable (possible timeseries?)\n' + ) + print(cube) + raise Exception("Variable can not be processed") + +# def fix_latlon_coord(cube, grid_type, dlat, dlon): +# def _add_coord_bounds(coord): +# if len(coord.points) > 1: +# if not coord.has_bounds(): +# coord.guess_bounds() +# else: +# # For length 1, assume it's global. guess_bounds doesn't work in this case +# if coord.name() == 'latitude': +# if not coord.has_bounds(): +# coord.bounds = np.array([[-90., 90.]]) +# elif coord.name() == 'longitude': +# if not coord.has_bounds(): +# coord.bounds = np.array([[0., 360.]]) + +# lat = cube.coord('latitude') + +# # Force to double for consistency with CMOR +# lat.points = lat.points.astype(np.float64) +# _add_coord_bounds(lat) +# lon = cube.coord('longitude') +# lon.points = lon.points.astype(np.float64) +# _add_coord_bounds(lon) # TODO: refactor to "rename level coord" @@ -440,13 +522,7 @@ def process(infile, outfile, args): # Interval in cell methods isn't reliable so better to remove it. c.cell_methods = fix_cell_methods(c.cell_methods) - try: - fix_latlon_coord(c, grid_type, dlat, dlon) - except iris.exceptions.CoordinateNotFoundError: - print( - '\nMissing lat/lon coordinates for variable (possible timeseries?)\n') - print(c) - raise Exception("Variable can not be processed") + fix_latlon_coords(c, grid_type, dlat, dlon) fix_level_coord(c, z_rho, z_theta) From 4545c5b81f67159b0f07f2f40d40ca9b55c99cbb Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Thu, 1 Aug 2024 01:10:08 +1000 Subject: [PATCH 03/27] split fix_latlon_names to separate functions and add tests --- test/test_um2netcdf.py | 113 ++++++++++++++++++++++++++++++++++------- umpost/um2netcdf.py | 88 +++++++++++++++++++------------- 2 files changed, 149 insertions(+), 52 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 49ff8a0..9760abe 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -412,10 +412,10 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units -def test_is_lat_river(): +def test_is_lat_river_grid(): lat_river_points = np.arange(-90., 90) + 0.5 assert len(lat_river_points) == 180 - assert um2nc.is_lat_river(lat_river_points) + assert um2nc.is_lat_river_grid(lat_river_points) # latitude points on ESM1.5 N96 not_lat_river_points = np.arange(-88.75, 89., 1.25) @@ -424,61 +424,61 @@ def test_is_lat_river(): [90.] )) assert len(not_lat_river_points) != 180 - assert not um2nc.is_lat_river(not_lat_river_points) + assert not um2nc.is_lat_river_grid(not_lat_river_points) -def test_is_lon_river(): +def test_is_lon_river_grid(): lon_river_points = np.arange(0., 360.) + 0.5 assert len(lon_river_points) == 360 - assert um2nc.is_lon_river(lon_river_points) + assert um2nc.is_lon_river_grid(lon_river_points) # longitude points on normal ESM1.5 N96 grid not_lon_river_points = np.arange(0., 360., 1.875) assert len(not_lon_river_points) != 360 - assert not um2nc.is_lon_river(not_lon_river_points) + assert not um2nc.is_lon_river_grid(not_lon_river_points) -def test_is_lat_v_EG(): +def test_is_lat_v_grid_EG(): dlat = 1.25 lat_v_points = np.arange(-90., 90, dlat) assert lat_v_points[0] == -90 - assert um2nc.is_lat_v(lat_v_points, um2nc.GRID_END_GAME, dlat) + assert um2nc.is_lat_v_grid(lat_v_points, um2nc.GRID_END_GAME, dlat) not_lat_v_points = np.arange(-88.5, 90, dlat) assert not_lat_v_points[0] != -90 - assert not um2nc.is_lat_v(not_lat_v_points, um2nc.GRID_END_GAME, dlat) + assert not um2nc.is_lat_v_grid(not_lat_v_points, um2nc.GRID_END_GAME, dlat) -def test_is_lat_v_NG(): +def test_is_lat_v_grid_ND(): dlat = 1.25 lat_v_points = np.arange(-90.+0.5*dlat, 90, dlat) assert lat_v_points[0] == -90.+0.5*dlat - assert um2nc.is_lat_v(lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) + assert um2nc.is_lat_v_grid(lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) not_lat_v_points = np.arange(-90., 90, dlat) - assert not um2nc.is_lat_v(not_lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) + assert not um2nc.is_lat_v_grid(not_lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) -def test_is_lon_u_EG(): +def test_is_lon_u_grid_EG(): dlon = 1.875 lon_u_points = np.arange(0, 360, dlon) assert lon_u_points[0] == 0 - assert um2nc.is_lon_u(lon_u_points, um2nc.GRID_END_GAME, dlon) + assert um2nc.is_lon_u_grid(lon_u_points, um2nc.GRID_END_GAME, dlon) not_lon_u_points = np.arange(0.5, 360, dlon) assert not_lon_u_points[0] != 0 - assert not um2nc.is_lon_u(not_lon_u_points, um2nc.GRID_END_GAME, dlon) + assert not um2nc.is_lon_u_grid(not_lon_u_points, um2nc.GRID_END_GAME, dlon) -def test_is_lon_u_ND(): +def test_is_lon_u_grid_ND(): dlon = 1.875 lon_u_points = np.arange(0.5*dlon, 360, dlon) assert lon_u_points[0] == 0.5*dlon - assert um2nc.is_lon_u(lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + assert um2nc.is_lon_u_grid(lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) not_lon_u_points = np.arange(0, 360, dlon) assert not_lon_u_points[0] != 0.5*dlon - assert not um2nc.is_lon_u(not_lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + assert not um2nc.is_lon_u_grid(not_lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) @dataclass @@ -489,6 +489,9 @@ class DummyCoordinate: coordname: str points: np.ndarray bounds: np.ndarray = None + # Note that var_name attribute is different thing to return + # value of name() method. + var_name: str = None def name(self): return self.coordname @@ -562,6 +565,80 @@ def test_add_latlon_coord_error(): with pytest.raises(ValueError): um2nc.add_latlon_coord_bounds(fake_coord) +def test_fix_lat_coord_name(): + # Following values are ignored due to mocking of checking functions. + grid_type = um2nc.GRID_END_GAME + dlat = 1.875 + lat_points = np.array([1.,2.,3.]) + + latitude_coordinate = DummyCoordinate( + um2nc.LAT_COORD_NAME, + lat_points + ) + assert latitude_coordinate.var_name is None + + # Mock the return value of grid checking functions in order to simplify test setup. + # Grid checking functions have their own tests. + with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = True): + um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) + assert latitude_coordinate.var_name == "lat_river" + + latitude_coordinate.var_name = None + with ( + mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = False), + mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value = True) + ): + um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) + assert latitude_coordinate.var_name == "lat_v" + + latitude_coordinate.var_name = None + with ( + mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = False), + mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value = False) + ): + um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) + assert latitude_coordinate.var_name == "lat" + + +# Probably better to fixture some realistic coordinates, which can also be used +# for the check function tests. + +def test_fix_lon_coord_name(): + # Following values are ignored due to mocking of checking functions. + grid_type = um2nc.GRID_END_GAME + dlon = 1.875 + lon_points = np.array([1.,2.,3.]) + + longitude_coordinate = DummyCoordinate( + um2nc.LON_COORD_NAME, + lon_points + ) + assert longitude_coordinate.var_name is None + + # Mock the return value of grid checking functions in order to simplify test setup. + # Grid checking functions have their own tests. + with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = True): + um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) + assert longitude_coordinate.var_name == "lon_river" + + longitude_coordinate.var_name = None + with ( + mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = False), + mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value = True) + ): + um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) + assert longitude_coordinate.var_name == "lon_u" + + longitude_coordinate.var_name = None + with ( + mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = False), + mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value = False) + ): + um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) + assert longitude_coordinate.var_name == "lat" + + + class DummyCubeWithCoords(DummyCube): # Dummy cube with coordinates, which can be filled with diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index bf10f75..d61656c 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -104,41 +104,62 @@ def convert_proleptic(time): time.units = newunits -def fix_latlon_coord_names(cube, grid_type, dlat, dlon): +def fix_lat_coord_name(lat_coordinate, grid_type, dlat): """ - Rename cube's latitude/longitude coordinate variables based on - the grid they lie on. + Add a 'var_name' attribute to a latitude coordinate object + based on the grid it lies on. Parameters ---------- - cube: an Iris cube object to modify (changes names in place). + lat_coordinate: coordinate object from iris cube (edits in place). grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. NB - Only applies to variables on the main horizontal grids, and not the river grid. + """ + + if lat_coordinate.name() != LAT_COORD_NAME: + raise ValueError( + f"Wrong coordinate {lat_coordinate.name()} supplied. " + f"Expected {LAT_COORD_NAME}." + ) + + if is_lat_river_grid(lat_coordinate.points): + lat_coordinate.var_name = 'lat_river' + elif is_lat_v_grid(lat_coordinate.points, grid_type, dlat): + lat_coordinate.var_name = 'lat_v' + else: + lat_coordinate.var_name = 'lat' + +def fix_lon_coord_name(lon_coordinate, grid_type, dlon): + """ + Add a 'var_name' attribute to a longitude coordinate object + based on the grid it lies on. + + Parameters + ---------- + lon_coordinate: coordinate object from iris cube (edits in place). + grid_type: (string) model horizontal grid type. dlon: (float) zonal spacing between longitude grid points. NB - Only applies to variables on the main horizontal grids, and not the river grid. """ - lat = cube.coord('latitude') - if is_lat_river(lat.points): - lat.var_name = 'lat_river' - elif is_lat_v(lat.points, grid_type, dlat): - lat.var_name = 'lat_v' - else: - lat.var_name = 'lat' - - lon = cube.coord('longitude') - if is_lon_river(lon.points): - lon.var_name = 'lon_river' - elif is_lon_u(lon.points, grid_type, dlon): - lon.var_name = 'lon_u' + if lon_coordinate.name() != LON_COORD_NAME: + raise ValueError( + f"Wrong coordinate {lon_coordinate.name()} supplied. " + f"Expected {LAT_COORD_NAME}." + ) + + if is_lon_river_grid(lon_coordinate.points): + lon_coordinate.var_name = 'lon_river' + elif is_lon_u_grid(lon_coordinate.points, grid_type, dlon): + lon_coordinate.var_name = 'lon_u' else: - lon.var_name = 'lon' + lon_coordinate.var_name = 'lon' -def is_lat_river(latitude_points): +def is_lat_river_grid(latitude_points): """ Check whether latitude points are on the river routing grid. @@ -149,7 +170,7 @@ def is_lat_river(latitude_points): return len(latitude_points) == 180 -def is_lon_river(longitude_points): +def is_lon_river_grid(longitude_points): """ Check whether longitude points are on the river routing grid. @@ -161,7 +182,7 @@ def is_lon_river(longitude_points): return len(longitude_points) == 360 -def is_lat_v(latitude_points, grid_type, dlat): +def is_lat_v_grid(latitude_points, grid_type, dlat): """ Check whether latitude points are on the lat_v grid. @@ -172,17 +193,18 @@ def is_lat_v(latitude_points, grid_type, dlat): dlat: (float) meridional spacing between latitude grid points. """ - first_latitude = latitude_points[0] + min_latitude = latitude_points[0] + min_latitude_v_ND = -90.+0.5*dlat return ( - (first_latitude == -90 and grid_type == GRID_END_GAME) or + (min_latitude == -90 and grid_type == GRID_END_GAME) or ( - np.allclose(-90.+0.5*dlat, first_latitude) and + np.allclose(min_latitude_v_ND, min_latitude) and grid_type == GRID_NEW_DYNAMICS ) ) -def is_lon_u(longitude_points, grid_type, dlon): +def is_lon_u_grid(longitude_points, grid_type, dlon): """ Check whether longitude points are on the lon_u grid. @@ -192,20 +214,18 @@ def is_lon_u(longitude_points, grid_type, dlon): grid_type: (string) model horizontal grid type. dlon: (float) zonal spacing between longitude grid points. """ - first_longitude = longitude_points[0] - - ND_first_lon_u = 0.5*dlon + min_longitude = longitude_points[0] + min_longitude_u_ND = 0.5*dlon return ( - (first_longitude == 0 and grid_type == GRID_END_GAME) or + (min_longitude == 0 and grid_type == GRID_END_GAME) or ( - np.allclose(ND_first_lon_u, first_longitude) and + np.allclose(min_longitude_u_ND, min_longitude) and grid_type == GRID_NEW_DYNAMICS ) ) - def add_latlon_coord_bounds(cube_coordinate): """ Add bounds to horizontal coordinate (longitude or latitude) if @@ -262,9 +282,9 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): add_latlon_coord_bounds(latitude_coordinate) add_latlon_coord_bounds(longitude_coordinate) - - # Coordinate names should only be changed after the bounds are added. - fix_latlon_coord_names(cube, grid_type, dlat, dlon) + + fix_lat_coord_name(latitude_coordinate, grid_type, dlat) + fix_lon_coord_name(longitude_coordinate, grid_type, dlon) except iris.exceptions.CoordinateNotFoundError: print( From 45dfa8e695b4a01ba362db37652091d632e9b244 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Thu, 1 Aug 2024 12:21:39 +1000 Subject: [PATCH 04/27] Add fixtures for lon/lat array examples and refactor tests --- test/test_um2netcdf.py | 208 +++++++++++++++++++++++++++++------------ 1 file changed, 150 insertions(+), 58 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 9760abe..a671d17 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -412,73 +412,145 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units -def test_is_lat_river_grid(): - lat_river_points = np.arange(-90., 90) + 0.5 +@pytest.fixture +def lat_river_points(): + # Array of points imitating real latitudes on UM V7.3's river grid. + # Must have length 180. + return np.arange(-90., 90) + 0.5 + + +@pytest.fixture +def lon_river_points(): + # Array of points imitating real longitudes on UM V7.3's river grid. + # Must have length 360. + return np.arange(0., 360.) + 0.5 + + +@pytest.fixture +def lat_v_points_dlat_ND(): + # Array of latitude points and corresponding spacing imitating the real + # lat_v grid from ESM1.5 (which uses the New Dynamics grid). + + # TODO: When gadi is back online, check that these match real lat_v + # points from ESM1.5 + + dlat = 1.25 + lat_v_points = np.arange(-90.+0.5*dlat, 90, dlat) + return (lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) + + +@pytest.fixture +def lon_u_points_dlon_ND(): + # Array of longitude points and corresponding spacing imitating the real + # lon_u grid from ESM1.5 (which uses the New Dynamics grid). + + # TODO: When gadi is back online, check that these match real lon_u + # points from ESM1.5 + dlon = 1.875 + lon_u_points = np.arange(0, 360, dlon) + + return (lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + + +@pytest.fixture +def lat_v_points_dlat_EG(): + # Array of latitude points and corresponding spacing imitating + # lat_v grid with grid type EG. + + # TODO: Find some CM2 output and use its values instead. + + dlat = 1.25 + lat_v_points = np.arange(-90., 90, dlat) + return (lat_v_points, um2nc.GRID_END_GAME, dlat) + + +@pytest.fixture +def lon_u_points_dlon_EG(): + # Array of longitude points and corresponding spacing imitating + # a lon_v grid with grid type EG. + + # TODO: Find some CM2 output and use its values instead. + dlon = 1.875 + lon_u_points = np.arange(0.5*dlon, 360, dlon) + + return (lon_u_points, um2nc.GRID_END_GAME, dlon) + + +@pytest.fixture +def lat_points_standard_dlat(): + # Array of latitude points and corresponding spacing and grid type + # on a standard (not river or v) grid. Copied from ESM1.5. + + # TODO: Once gadi back, confirm these values are correct. + dlat = 1.25 + lat_points_middle = np.arange(-88.75, 89., 1.25) + lat_points = np.concatenate(([-90], + lat_points_middle, + [90.] + )) + return (lat_points, um2nc.GRID_NEW_DYNAMICS, dlat) + +@pytest.fixture +def lon_points_standard_dlon(): + # Array of longitude points and corresponding spacing and grid type + # on a standard (not river or u) grid. Copied from ESM1.5. + + # TODO: Once gadi back, confirm these values are correct. + dlon = 1.875 + lon_points = np.arange(0, 360, dlon) + + return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) + + + + +def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat): assert len(lat_river_points) == 180 assert um2nc.is_lat_river_grid(lat_river_points) # latitude points on ESM1.5 N96 - not_lat_river_points = np.arange(-88.75, 89., 1.25) - not_lat_river_points = np.concatenate(([-90], - not_lat_river_points, - [90.] - )) + not_lat_river_points = lat_points_standard_dlat[0] + assert len(not_lat_river_points) != 180 assert not um2nc.is_lat_river_grid(not_lat_river_points) -def test_is_lon_river_grid(): - lon_river_points = np.arange(0., 360.) + 0.5 +def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon): assert len(lon_river_points) == 360 assert um2nc.is_lon_river_grid(lon_river_points) # longitude points on normal ESM1.5 N96 grid - not_lon_river_points = np.arange(0., 360., 1.875) + not_lon_river_points = lon_points_standard_dlon[0] assert len(not_lon_river_points) != 360 assert not um2nc.is_lon_river_grid(not_lon_river_points) -def test_is_lat_v_grid_EG(): - dlat = 1.25 - lat_v_points = np.arange(-90., 90, dlat) - assert lat_v_points[0] == -90 - assert um2nc.is_lat_v_grid(lat_v_points, um2nc.GRID_END_GAME, dlat) - - not_lat_v_points = np.arange(-88.5, 90, dlat) - assert not_lat_v_points[0] != -90 - assert not um2nc.is_lat_v_grid(not_lat_v_points, um2nc.GRID_END_GAME, dlat) - +def test_is_lat_v_grid(lat_v_points_dlat_EG, + lat_v_points_dlat_ND, + lat_points_standard_dlat + ): + lat_v_points, grid_code, dlat = lat_v_points_dlat_EG + assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) -def test_is_lat_v_grid_ND(): - dlat = 1.25 - lat_v_points = np.arange(-90.+0.5*dlat, 90, dlat) - assert lat_v_points[0] == -90.+0.5*dlat - assert um2nc.is_lat_v_grid(lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) + lat_v_points, grid_code, dlat = lat_v_points_dlat_ND + assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) - not_lat_v_points = np.arange(-90., 90, dlat) - assert not um2nc.is_lat_v_grid(not_lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) - - -def test_is_lon_u_grid_EG(): - dlon = 1.875 - lon_u_points = np.arange(0, 360, dlon) - assert lon_u_points[0] == 0 - assert um2nc.is_lon_u_grid(lon_u_points, um2nc.GRID_END_GAME, dlon) + not_lat_v_points, grid_code, dlat = lat_points_standard_dlat + assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) - not_lon_u_points = np.arange(0.5, 360, dlon) - assert not_lon_u_points[0] != 0 - assert not um2nc.is_lon_u_grid(not_lon_u_points, um2nc.GRID_END_GAME, dlon) +def test_is_lon_u_grid(lon_u_points_dlon_EG, + lon_u_points_dlon_ND, + lon_points_standard_dlon + ): + lon_u_points, grid_code, dlon = lon_u_points_dlon_EG + assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) -def test_is_lon_u_grid_ND(): - dlon = 1.875 - lon_u_points = np.arange(0.5*dlon, 360, dlon) - assert lon_u_points[0] == 0.5*dlon - assert um2nc.is_lon_u_grid(lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + lon_u_points, grid_code, dlon = lon_u_points_dlon_ND + assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) - not_lon_u_points = np.arange(0, 360, dlon) - assert not_lon_u_points[0] != 0.5*dlon - assert not um2nc.is_lon_u_grid(not_lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) + not_lon_u_points, grid_code, dlon = lon_points_standard_dlon + assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) @dataclass @@ -537,6 +609,8 @@ def test_add_latlon_coord_guess_bounds(): def test_add_latlon_coord_single(): + # Test that the correct global bounds are added to coordinates + # with just a single point. for coord_name in [um2nc.LON_COORD_NAME, um2nc.LAT_COORD_NAME]: points = np.array([0.]) coord_single_point = DummyCoordinate( @@ -577,8 +651,8 @@ def test_fix_lat_coord_name(): ) assert latitude_coordinate.var_name is None - # Mock the return value of grid checking functions in order to simplify test setup. - # Grid checking functions have their own tests. + # Mock the return value of grid checking functions in order to simplify test setup, + # since grid checking functions have their own tests. with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = True): um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) assert latitude_coordinate.var_name == "lat_river" @@ -600,9 +674,6 @@ def test_fix_lat_coord_name(): assert latitude_coordinate.var_name == "lat" -# Probably better to fixture some realistic coordinates, which can also be used -# for the check function tests. - def test_fix_lon_coord_name(): # Following values are ignored due to mocking of checking functions. grid_type = um2nc.GRID_END_GAME @@ -615,8 +686,8 @@ def test_fix_lon_coord_name(): ) assert longitude_coordinate.var_name is None - # Mock the return value of grid checking functions in order to simplify test setup. - # Grid checking functions have their own tests. + # Mock the return value of grid checking functions in order to simplify test setup, + # since grid checking functions have their own tests. with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = True): um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) assert longitude_coordinate.var_name == "lon_river" @@ -635,13 +706,36 @@ def test_fix_lon_coord_name(): mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value = False) ): um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) - assert longitude_coordinate.var_name == "lat" + assert longitude_coordinate.var_name == "lon" +@pytest.fixture +def coordinate_fake_name(): + # Fake dummy coordinate with made up name to test that exceptions are raised + fake_coord_name = "fake coordinate" + fake_points = np.array([1., 2., 3.]) + + fake_coord = DummyCoordinate( + fake_coord_name, + fake_points + ) + + return fake_coord + + +def test_fix_lat_coord_name_error(coordinate_fake_name): + with pytest.raises("ValueError"): + um2nc.fix_lat_coord_name(coordinate_fake_name) + + +def test_fix_lon_coord_name_error(coordinate_fake_name): + with pytest.raises("ValueError"): + um2nc.fix_lon_coord_name(coordinate_fake_name) + class DummyCubeWithCoords(DummyCube): - # Dummy cube with coordinates, which can be filled with + # DummyCube with coordinates, which can be filled with # DummyCoordinate objects for testing. def __init__(self, item_code, var_name=None, attributes=None, units=None, coords = {}): super().__init__(item_code, var_name, attributes, units) @@ -653,7 +747,7 @@ def coord(self, coordinate_name): @pytest.fixture -def cube_with_latlon_coords(): +def ua_plev_cube_with_latlon_coords(): lat_points = np.array([-90., -88.75, -87.5 ], dtype = "float32") lon_points = np.array([ 0., 1.875, 3.75 ], dtype = "float32") @@ -671,9 +765,7 @@ def cube_with_latlon_coords(): [um2nc.LON_COORD_NAME]: lon_coord_object } + cube_with_coords = DummyCubeWithCoords() - -# TODO test_fix_latlon_coord_names(): - # TODO test_fix_latlon_coords(): \ No newline at end of file From 468e43409d623c3b2366fc72a331d75bc00c2c2b Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Thu, 1 Aug 2024 16:39:19 +1000 Subject: [PATCH 05/27] Clean up fix_latlon exception handling and add tests --- test/test_um2netcdf.py | 36 +++++++++++++++++++++++++++++++++++- umpost/um2netcdf.py | 33 +++++---------------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index a671d17..a9e8ccf 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from collections import namedtuple import numpy as np +from iris.exceptions import CoordinateNotFoundError import umpost.um2netcdf as um2nc @@ -768,4 +769,37 @@ def ua_plev_cube_with_latlon_coords(): cube_with_coords = DummyCubeWithCoords() -# TODO test_fix_latlon_coords(): \ No newline at end of file +def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): + # Test that coordinate arrays are converted to float64 + + # Following values don't matter for test. Just needed as arguments + grid_type = um2nc.GRID_NEW_DYNAMICS + dlat = 1.25 + dlon = 1.875 + + lat_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LAT_COORD_NAME) + lon_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LON_COORD_NAME) + + assert lat_coord.points.dtype == np.dtype("float32") + assert lon_coord.points.dtype == np.dtype("float32") + + um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) + + assert lat_coord.points.dtype == np.dtype("float64") + assert lon_coord.points.dtype == np.dtype("float64") + +# TODO: actually test this. +def test_fix_latlon_coords_error(ua_plev_cube_with_latlon_coords): + # Test that fix_latlon_coords raises the right type of error when a cube + # is missing coordinates + def _raise_CoordinateNotFoundError(coord_name): + raise CoordinateNotFoundError() + + # Replace coord method to raise CoordinateNotFoundError + ua_plev_cube_with_latlon_coords.coord = _raise_CoordinateNotFoundError + + with ( + pytest.warns(), + pytest.raises(CoordinateNotFoundError) + ): + um2nc.fix_lat_lon_coords(ua_plev_cube_with_latlon_coords) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index d61656c..6140462 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -287,34 +287,11 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): fix_lon_coord_name(longitude_coordinate, grid_type, dlon) except iris.exceptions.CoordinateNotFoundError: - print( - '\nMissing lat/lon coordinates for variable (possible timeseries?)\n' - ) - print(cube) - raise Exception("Variable can not be processed") - -# def fix_latlon_coord(cube, grid_type, dlat, dlon): -# def _add_coord_bounds(coord): -# if len(coord.points) > 1: -# if not coord.has_bounds(): -# coord.guess_bounds() -# else: -# # For length 1, assume it's global. guess_bounds doesn't work in this case -# if coord.name() == 'latitude': -# if not coord.has_bounds(): -# coord.bounds = np.array([[-90., 90.]]) -# elif coord.name() == 'longitude': -# if not coord.has_bounds(): -# coord.bounds = np.array([[0., 360.]]) - -# lat = cube.coord('latitude') - -# # Force to double for consistency with CMOR -# lat.points = lat.points.astype(np.float64) -# _add_coord_bounds(lat) -# lon = cube.coord('longitude') -# lon.points = lon.points.astype(np.float64) -# _add_coord_bounds(lon) + warnings.warn( + "Missing latitude or longitude coordinate for variable (possible timeseries?): \n" + f"{cube}\n" + ) + raise # TODO: refactor to "rename level coord" From cbc143b8f51ef9fb256fa5a2040051ca116d799d Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 2 Aug 2024 11:05:48 +1000 Subject: [PATCH 06/27] Fix broken tests --- test/test_um2netcdf.py | 98 ++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index a9e8ccf..5fbec30 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -442,13 +442,12 @@ def lat_v_points_dlat_ND(): @pytest.fixture def lon_u_points_dlon_ND(): - # Array of longitude points and corresponding spacing imitating the real + # Array of latitude points and corresponding spacing imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). - # TODO: When gadi is back online, check that these match real lon_u - # points from ESM1.5 + # TODO: Find some CM2 output and use its values instead. dlon = 1.875 - lon_u_points = np.arange(0, 360, dlon) + lon_u_points = np.arange(0.5*dlon, 360, dlon) return (lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) @@ -467,18 +466,19 @@ def lat_v_points_dlat_EG(): @pytest.fixture def lon_u_points_dlon_EG(): - # Array of longitude points and corresponding spacing imitating - # a lon_v grid with grid type EG. + # Array of longitude points and corresponding spacing imitating + # a lon_u grid with grid type EG. - # TODO: Find some CM2 output and use its values instead. + # TODO: When gadi is back online, check that these match real lon_u + # points from ESM1.5 dlon = 1.875 - lon_u_points = np.arange(0.5*dlon, 360, dlon) + lon_u_points = np.arange(0, 360, dlon) return (lon_u_points, um2nc.GRID_END_GAME, dlon) @pytest.fixture -def lat_points_standard_dlat(): +def lat_points_standard_dlat_ND(): # Array of latitude points and corresponding spacing and grid type # on a standard (not river or v) grid. Copied from ESM1.5. @@ -491,8 +491,9 @@ def lat_points_standard_dlat(): )) return (lat_points, um2nc.GRID_NEW_DYNAMICS, dlat) + @pytest.fixture -def lon_points_standard_dlon(): +def lon_points_standard_dlon_ND(): # Array of longitude points and corresponding spacing and grid type # on a standard (not river or u) grid. Copied from ESM1.5. @@ -503,32 +504,30 @@ def lon_points_standard_dlon(): return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) - - -def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat): +def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): assert len(lat_river_points) == 180 assert um2nc.is_lat_river_grid(lat_river_points) # latitude points on ESM1.5 N96 - not_lat_river_points = lat_points_standard_dlat[0] + not_lat_river_points = lat_points_standard_dlat_ND[0] assert len(not_lat_river_points) != 180 assert not um2nc.is_lat_river_grid(not_lat_river_points) -def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon): +def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): assert len(lon_river_points) == 360 assert um2nc.is_lon_river_grid(lon_river_points) # longitude points on normal ESM1.5 N96 grid - not_lon_river_points = lon_points_standard_dlon[0] + not_lon_river_points = lon_points_standard_dlon_ND[0] assert len(not_lon_river_points) != 360 assert not um2nc.is_lon_river_grid(not_lon_river_points) def test_is_lat_v_grid(lat_v_points_dlat_EG, lat_v_points_dlat_ND, - lat_points_standard_dlat + lat_points_standard_dlat_ND ): lat_v_points, grid_code, dlat = lat_v_points_dlat_EG assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) @@ -536,13 +535,13 @@ def test_is_lat_v_grid(lat_v_points_dlat_EG, lat_v_points, grid_code, dlat = lat_v_points_dlat_ND assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) - not_lat_v_points, grid_code, dlat = lat_points_standard_dlat + not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_ND assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) def test_is_lon_u_grid(lon_u_points_dlon_EG, lon_u_points_dlon_ND, - lon_points_standard_dlon + lon_points_standard_dlon_ND ): lon_u_points, grid_code, dlon = lon_u_points_dlon_EG assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) @@ -550,7 +549,7 @@ def test_is_lon_u_grid(lon_u_points_dlon_EG, lon_u_points, grid_code, dlon = lon_u_points_dlon_ND assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) - not_lon_u_points, grid_code, dlon = lon_points_standard_dlon + not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_ND assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) @@ -640,6 +639,7 @@ def test_add_latlon_coord_error(): with pytest.raises(ValueError): um2nc.add_latlon_coord_bounds(fake_coord) + def test_fix_lat_coord_name(): # Following values are ignored due to mocking of checking functions. grid_type = um2nc.GRID_END_GAME @@ -710,7 +710,6 @@ def test_fix_lon_coord_name(): assert longitude_coordinate.var_name == "lon" - @pytest.fixture def coordinate_fake_name(): # Fake dummy coordinate with made up name to test that exceptions are raised @@ -726,13 +725,19 @@ def coordinate_fake_name(): def test_fix_lat_coord_name_error(coordinate_fake_name): - with pytest.raises("ValueError"): - um2nc.fix_lat_coord_name(coordinate_fake_name) + # Following values unimportant. Just needed as function arguments. + grid_type = um2nc.GRID_NEW_DYNAMICS + dlat = 1.25 + with pytest.raises(ValueError): + um2nc.fix_lat_coord_name(coordinate_fake_name, grid_type, dlat) def test_fix_lon_coord_name_error(coordinate_fake_name): - with pytest.raises("ValueError"): - um2nc.fix_lon_coord_name(coordinate_fake_name) + # Following values unimportant. Just needed as function arguments. + grid_type = um2nc.GRID_NEW_DYNAMICS + dlon = 1.875 + with pytest.raises(ValueError): + um2nc.fix_lon_coord_name(coordinate_fake_name, grid_type, dlon) class DummyCubeWithCoords(DummyCube): @@ -744,11 +749,10 @@ def __init__(self, item_code, var_name=None, attributes=None, units=None, coords def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] - - + @pytest.fixture -def ua_plev_cube_with_latlon_coords(): +def ua_plev_cube_with_latlon_coords(ua_plev_cube): lat_points = np.array([-90., -88.75, -87.5 ], dtype = "float32") lon_points = np.array([ 0., 1.875, 3.75 ], dtype = "float32") @@ -762,11 +766,17 @@ def ua_plev_cube_with_latlon_coords(): ) coords_dict = { - [um2nc.LAT_COORD_NAME]: lat_coord_object, - [um2nc.LON_COORD_NAME]: lon_coord_object + um2nc.LAT_COORD_NAME: lat_coord_object, + um2nc.LON_COORD_NAME: lon_coord_object } - cube_with_coords = DummyCubeWithCoords() + cube_with_coords = DummyCubeWithCoords( + ua_plev_cube.item_code, + ua_plev_cube.var_name, + coords = coords_dict + ) + + return cube_with_coords def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): @@ -782,24 +792,40 @@ def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): assert lat_coord.points.dtype == np.dtype("float32") assert lon_coord.points.dtype == np.dtype("float32") + - um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) + # Mock additional functions called by um2nc.fix_latlon_coords, as they may + # require methods not implemented by the DummyCubeWithCoordinates. + with ( + mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value = None), + mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value = None), + mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value = None), + ): + um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) assert lat_coord.points.dtype == np.dtype("float64") assert lon_coord.points.dtype == np.dtype("float64") -# TODO: actually test this. + + def test_fix_latlon_coords_error(ua_plev_cube_with_latlon_coords): # Test that fix_latlon_coords raises the right type of error when a cube # is missing coordinates def _raise_CoordinateNotFoundError(coord_name): - raise CoordinateNotFoundError() + # Include an argument "coord_name" to mimic the signature of an + # Iris cube's ".coord" method + raise CoordinateNotFoundError(coord_name) + + # Following values don't matter for test. Just needed as arguments + grid_type = um2nc.GRID_NEW_DYNAMICS + dlat = 1.25 + dlon = 1.875 # Replace coord method to raise CoordinateNotFoundError ua_plev_cube_with_latlon_coords.coord = _raise_CoordinateNotFoundError - + with ( pytest.warns(), pytest.raises(CoordinateNotFoundError) ): - um2nc.fix_lat_lon_coords(ua_plev_cube_with_latlon_coords) + um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) From b2fb618221ca41c636eb2c69d141d107fcc26b80 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 2 Aug 2024 12:06:58 +1000 Subject: [PATCH 07/27] Update coordinate array fixtures to match real coordinated from ESM1.5 and CM2 --- test/test_um2netcdf.py | 66 +++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 5fbec30..69722f9 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -431,9 +431,6 @@ def lon_river_points(): def lat_v_points_dlat_ND(): # Array of latitude points and corresponding spacing imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). - - # TODO: When gadi is back online, check that these match real lat_v - # points from ESM1.5 dlat = 1.25 lat_v_points = np.arange(-90.+0.5*dlat, 90, dlat) @@ -445,7 +442,6 @@ def lon_u_points_dlon_ND(): # Array of latitude points and corresponding spacing imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). - # TODO: Find some CM2 output and use its values instead. dlon = 1.875 lon_u_points = np.arange(0.5*dlon, 360, dlon) @@ -455,34 +451,29 @@ def lon_u_points_dlon_ND(): @pytest.fixture def lat_v_points_dlat_EG(): # Array of latitude points and corresponding spacing imitating - # lat_v grid with grid type EG. - - # TODO: Find some CM2 output and use its values instead. + # the real lat_v grid from CM2 which uses grid type EG. dlat = 1.25 - lat_v_points = np.arange(-90., 90, dlat) + lat_v_points = np.arange(-90., 91, dlat) return (lat_v_points, um2nc.GRID_END_GAME, dlat) @pytest.fixture def lon_u_points_dlon_EG(): - # Array of longitude points and corresponding spacing imitating - # a lon_u grid with grid type EG. + # Array of longitude points and corresponding spacing imitating + # the real lon_u grid from CM2 which uses grid type EG. - # TODO: When gadi is back online, check that these match real lon_u - # points from ESM1.5 dlon = 1.875 lon_u_points = np.arange(0, 360, dlon) - return (lon_u_points, um2nc.GRID_END_GAME, dlon) @pytest.fixture def lat_points_standard_dlat_ND(): - # Array of latitude points and corresponding spacing and grid type - # on a standard (not river or v) grid. Copied from ESM1.5. + # Array of latitude points and corresponding spacing imitating the + # standard (not river or v) lat grid for ESM1.5 which uses + # grid type ND. - # TODO: Once gadi back, confirm these values are correct. dlat = 1.25 lat_points_middle = np.arange(-88.75, 89., 1.25) lat_points = np.concatenate(([-90], @@ -494,16 +485,37 @@ def lat_points_standard_dlat_ND(): @pytest.fixture def lon_points_standard_dlon_ND(): - # Array of longitude points and corresponding spacing and grid type - # on a standard (not river or u) grid. Copied from ESM1.5. + # Array of longitude points and corresponding spacing imitating the + # standard (not river or u) lon grid for ESM1.5 which uses grid + # type ND. - # TODO: Once gadi back, confirm these values are correct. dlon = 1.875 lon_points = np.arange(0, 360, dlon) - return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) +@pytest.fixture +def lat_points_standard_dlat_EG(): + # Array of latitude points and corresponding spacing imitating the + # standard (not river or v) lat grid for CM2 which uses + # grid type EG. + + dlat = 1.25 + lat_points = np.arange(-90 + 0.5*dlat, 90., dlat) + return (lat_points, um2nc.GRID_END_GAME, dlat) + + +@pytest.fixture +def lon_points_standard_dlon_EG(): + # Array of longitude points and corresponding spacing imitating the + # standard (not river or u) lon grid for CM2 which uses grid + # type EG. + + dlon = 1.875 + lon_points = np.arange(0.5*dlon, 360, dlon) + return (lon_points, um2nc.GRID_END_GAME, dlon) + + def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): assert len(lat_river_points) == 180 assert um2nc.is_lat_river_grid(lat_river_points) @@ -527,7 +539,8 @@ def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): def test_is_lat_v_grid(lat_v_points_dlat_EG, lat_v_points_dlat_ND, - lat_points_standard_dlat_ND + lat_points_standard_dlat_ND, + lat_points_standard_dlat_EG ): lat_v_points, grid_code, dlat = lat_v_points_dlat_EG assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) @@ -535,13 +548,17 @@ def test_is_lat_v_grid(lat_v_points_dlat_EG, lat_v_points, grid_code, dlat = lat_v_points_dlat_ND assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) + not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_EG + assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) + not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_ND assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) def test_is_lon_u_grid(lon_u_points_dlon_EG, lon_u_points_dlon_ND, - lon_points_standard_dlon_ND + lon_points_standard_dlon_ND, + lon_points_standard_dlon_EG ): lon_u_points, grid_code, dlon = lon_u_points_dlon_EG assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) @@ -549,6 +566,9 @@ def test_is_lon_u_grid(lon_u_points_dlon_EG, lon_u_points, grid_code, dlon = lon_u_points_dlon_ND assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) + not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_EG + assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) + not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_ND assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) @@ -799,7 +819,7 @@ def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): with ( mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value = None), mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value = None), - mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value = None), + mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value = None) ): um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) From 995345c2f4150604c796c67b806f37cfa8ae2218 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 2 Aug 2024 12:24:15 +1000 Subject: [PATCH 08/27] Try undo accidental autoformat... --- umpost/um2netcdf.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 6140462..fd20dc8 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -77,10 +77,8 @@ def pg_calendar(self): # TODO: rename time to avoid clash with builtin time module def convert_proleptic(time): # Convert units from hours to days and shift origin from 1970 to 0001 - newunits = cf_units.Unit( - "days since 0001-01-01 00:00", calendar='proleptic_gregorian') - # Need a copy because can't assign to time.points[i] - tvals = np.array(time.points) + newunits = cf_units.Unit("days since 0001-01-01 00:00", calendar='proleptic_gregorian') + tvals = np.array(time.points) # Need a copy because can't assign to time.points[i] tbnds = np.array(time.bounds) if time.bounds is not None else None for i in range(len(time.points)): From cf70d714920d6ac80b4e757cf6a324e0fdef4739 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 2 Aug 2024 12:31:08 +1000 Subject: [PATCH 09/27] Continue attempting to undo accidental auto format --- umpost/um2netcdf.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index fd20dc8..5182179 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -344,8 +344,7 @@ def cubewrite(cube, sman, compression, use64bit, verbose): fill_value = 1.e20 else: # Use netCDF defaults - fill_value = default_fillvals['%s%1d' % ( - cube.data.dtype.kind, cube.data.dtype.itemsize)] + fill_value = default_fillvals['%s%1d' % (cube.data.dtype.kind, cube.data.dtype.itemsize)] cube.attributes['missing_value'] = np.array([fill_value], cube.data.dtype) @@ -365,8 +364,7 @@ def cubewrite(cube, sman, compression, use64bit, verbose): else: new_calendar = time.units.calendar - time.units = cf_units.Unit( - "days since 1970-01-01 00:00", calendar=new_calendar) + time.units = cf_units.Unit("days since 1970-01-01 00:00", calendar=new_calendar) time.points = time.points/24. if time.bounds is not None: @@ -415,8 +413,7 @@ def cubewrite(cube, sman, compression, use64bit, verbose): except iris.exceptions.CoordinateNotFoundError: # No time dimension (probably ancillary file) - sman.write(cube, zlib=True, complevel=compression, - fill_value=fill_value) + sman.write(cube, zlib=True, complevel=compression, fill_value=fill_value) def fix_cell_methods(mtuple): @@ -441,8 +438,7 @@ def apply_mask(c, heaviside, hcrit): # Temporarily turn off warnings from 0/0 # TODO: refactor to use np.where() with np.errstate(divide='ignore', invalid='ignore'): - c.data = np.ma.masked_array( - c.data/heaviside.data, heaviside.data <= hcrit).astype(np.float32) + c.data = np.ma.masked_array(c.data/heaviside.data, heaviside.data <= hcrit).astype(np.float32) else: # Are the levels of c a subset of the levels of the heaviside variable? c_p = c.coord('pressure') @@ -454,14 +450,11 @@ def apply_mask(c, heaviside, hcrit): h_tmp = heaviside.extract(constraint) # Double check they're actually the same after extraction if not np.all(c_p.points == h_tmp.coord('pressure').points): - raise Exception( - 'Unexpected mismatch in levels of extracted heaviside function') + raise Exception('Unexpected mismatch in levels of extracted heaviside function') with np.errstate(divide='ignore', invalid='ignore'): - c.data = np.ma.masked_array( - c.data/h_tmp.data, h_tmp.data <= hcrit).astype(np.float32) + c.data = np.ma.masked_array(c.data/h_tmp.data, h_tmp.data <= hcrit).astype(np.float32) else: - raise Exception( - 'Unable to match levels of heaviside function to variable %s' % c.name()) + raise Exception('Unable to match levels of heaviside function to variable %s' % c.name()) def process(infile, outfile, args): @@ -470,8 +463,7 @@ def process(infile, outfile, args): ff = mule.load_umfile(str(infile)) if isinstance(ff, mule.ancil.AncilFile): - raise NotImplementedError( - 'Ancillary files are currently not supported') + raise NotImplementedError('Ancillary files are currently not supported') # TODO: eventually move these calls closer to their usage grid_type = get_grid_type(ff) @@ -560,8 +552,7 @@ def get_grid_type(ff): elif staggering == 3: return GRID_NEW_DYNAMICS else: - raise PostProcessingError( - f"Unable to determine grid staggering from header '{staggering}'") + raise PostProcessingError(f"Unable to determine grid staggering from header '{staggering}'") def get_grid_spacing(ff): @@ -841,8 +832,7 @@ def fix_units(cube, um_var_units, verbose: bool): if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Convert UM fieldsfile to netcdf") + parser = argparse.ArgumentParser(description="Convert UM fieldsfile to netcdf") parser.add_argument('-k', dest='nckind', required=False, type=int, default=3, help=('specify netCDF output format: 1 classic, 2 64-bit' From 2e77e152d0b37a1027b927055b917dbb6eb1151c Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Mon, 12 Aug 2024 22:40:38 +1000 Subject: [PATCH 10/27] Review fixes inc magic numbers, readability, styling --- test/test_um2netcdf.py | 8 ++--- umpost/um2netcdf.py | 69 ++++++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 69722f9..64c9e34 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -517,23 +517,23 @@ def lon_points_standard_dlon_EG(): def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): - assert len(lat_river_points) == 180 + assert len(lat_river_points) == um2nc.NUM_LAT_RIVER_GRID assert um2nc.is_lat_river_grid(lat_river_points) # latitude points on ESM1.5 N96 not_lat_river_points = lat_points_standard_dlat_ND[0] - assert len(not_lat_river_points) != 180 + assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID assert not um2nc.is_lat_river_grid(not_lat_river_points) def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): - assert len(lon_river_points) == 360 + assert len(lon_river_points) == um2nc.NUM_LON_RIVER_GRID assert um2nc.is_lon_river_grid(lon_river_points) # longitude points on normal ESM1.5 N96 grid not_lon_river_points = lon_points_standard_dlon_ND[0] - assert len(not_lon_river_points) != 360 + assert len(not_lon_river_points) != um2nc.NUM_LON_RIVER_GRID assert not um2nc.is_lon_river_grid(not_lon_river_points) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 5182179..3f0e7a8 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -48,6 +48,9 @@ LON_COORD_NAME: np.array([[0., 360.]]), LAT_COORD_NAME: np.array([[-90., 90.]]) } + +NUM_LAT_RIVER_GRID = 180 +NUM_LON_RIVER_GRID = 360 @@ -165,7 +168,7 @@ def is_lat_river_grid(latitude_points): ---------- latitude_points: (array) 1D array of latitude grid points. """ - return len(latitude_points) == 180 + return len(latitude_points) == NUM_LAT_RIVER_GRID def is_lon_river_grid(longitude_points): @@ -177,7 +180,7 @@ def is_lon_river_grid(longitude_points): longitude_points: (array) 1D array of longitude grid points. """ - return len(longitude_points) == 360 + return len(longitude_points) == NUM_LON_RIVER_GRID def is_lat_v_grid(latitude_points, grid_type, dlat): @@ -192,11 +195,12 @@ def is_lat_v_grid(latitude_points, grid_type, dlat): """ min_latitude = latitude_points[0] - min_latitude_v_ND = -90.+0.5*dlat + min_lat_v_nd_grid = -90.+0.5*dlat + min_lat_v_eg_grid = -90 return ( - (min_latitude == -90 and grid_type == GRID_END_GAME) or + (min_latitude == min_lat_v_eg_grid and grid_type == GRID_END_GAME) or ( - np.allclose(min_latitude_v_ND, min_latitude) and + np.allclose(min_lat_v_nd_grid, min_latitude) and grid_type == GRID_NEW_DYNAMICS ) ) @@ -208,17 +212,18 @@ def is_lon_u_grid(longitude_points, grid_type, dlon): Parameters ---------- - longitude: (array) 1D array of longitude grid points. + longitude_points: (array) 1D array of longitude grid points. grid_type: (string) model horizontal grid type. dlon: (float) zonal spacing between longitude grid points. """ min_longitude = longitude_points[0] - min_longitude_u_ND = 0.5*dlon + min_lon_u_nd_grid = 0.5*dlon + min_lon_u_eg_grid = 0 return ( - (min_longitude == 0 and grid_type == GRID_END_GAME) or + (min_longitude == min_lon_u_eg_grid and grid_type == GRID_END_GAME) or ( - np.allclose(min_longitude_u_ND, min_longitude) and + np.allclose(min_lon_u_nd_grid, min_longitude) and grid_type == GRID_NEW_DYNAMICS ) ) @@ -240,16 +245,13 @@ def add_latlon_coord_bounds(cube_coordinate): f"Expected one of {LON_COORD_NAME}, {LAT_COORD_NAME}." ) - if cube_coordinate.has_bounds(): - # Don't change bounds if already exist - return - - if len(cube_coordinate.points) > 1: - cube_coordinate.guess_bounds() - else: - # For length 1, assume it's global. guess_bounds doesn't work in this case - cube_coordinate.bounds = GLOBAL_COORD_BOUNDS[coordinate_name] - + # Only add bounds if not already present. + if not cube_coordinate.has_bounds(): + if len(cube_coordinate.points) > 1: + cube_coordinate.guess_bounds() + else: + # For length 1, assume it's global. guess_bounds doesn't work in this case. + cube_coordinate.bounds = GLOBAL_COORD_BOUNDS[coordinate_name] def fix_latlon_coords(cube, grid_type, dlat, dlon): @@ -257,39 +259,40 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): Wrapper function to modify cube's horizontal coordinates (latitude and longitude). Converts to float64, adds grid bounds, and renames coordinates. Modifies cube in place. + + NB - grid spacings dlat and dlon only refer to variables on the main + horizontal grids, and not the river grid. Parameters ---------- cube: Iris cube object (modified in place). grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. - NB - Only applies to variables on the main horizontal grids, - and not the river grid. dlon: (float) zonal spacing between longitude grid points. - NB - Only applies to variables on the main horizontal grids, - and not the river grid. """ # TODO: Check that we don't need the double coordinate lookup. try: latitude_coordinate = cube.coord(LAT_COORD_NAME) longitude_coordinate = cube.coord(LON_COORD_NAME) - # Force to double for consistency with CMOR - latitude_coordinate.points = latitude_coordinate.points.astype(np.float64) - longitude_coordinate.points = longitude_coordinate.points.astype(np.float64) - - add_latlon_coord_bounds(latitude_coordinate) - add_latlon_coord_bounds(longitude_coordinate) - - fix_lat_coord_name(latitude_coordinate, grid_type, dlat) - fix_lon_coord_name(longitude_coordinate, grid_type, dlon) - except iris.exceptions.CoordinateNotFoundError: warnings.warn( "Missing latitude or longitude coordinate for variable (possible timeseries?): \n" f"{cube}\n" ) raise + + # Force to double for consistency with CMOR + latitude_coordinate.points = latitude_coordinate.points.astype(np.float64) + longitude_coordinate.points = longitude_coordinate.points.astype(np.float64) + + add_latlon_coord_bounds(latitude_coordinate) + add_latlon_coord_bounds(longitude_coordinate) + + fix_lat_coord_name(latitude_coordinate, grid_type, dlat) + fix_lon_coord_name(longitude_coordinate, grid_type, dlon) + + # TODO: refactor to "rename level coord" From 193d7f14fa1de131d5b7ebae0b69ca918c5945fe Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Wed, 14 Aug 2024 23:00:05 +1000 Subject: [PATCH 11/27] Review style changes, fix tests --- test/test_um2netcdf.py | 58 +++++++++++++++--------------- umpost/um2netcdf.py | 81 +++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 65 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index c0567a2..fec6ac3 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -149,7 +149,7 @@ def test_process_no_heaviside_drop_cubes(air_temp_cube, precipitation_flux_cube, # TODO: lat/long & level coord fixes require more internal data attrs # skip temporarily to manage test complexity - mock.patch("umpost.um2netcdf.fix_latlon_coord"), + mock.patch("umpost.um2netcdf.fix_latlon_coords"), mock.patch("umpost.um2netcdf.fix_level_coord"), mock.patch("umpost.um2netcdf.cubewrite"), ): @@ -208,7 +208,7 @@ def test_process_mask_with_heaviside(air_temp_cube, precipitation_flux_cube, mock.patch("iris.load") as m_iris_load, mock.patch("iris.fileformats.netcdf.Saver") as m_saver, # prevent I/O - mock.patch("umpost.um2netcdf.fix_latlon_coord"), + mock.patch("umpost.um2netcdf.fix_latlon_coords"), mock.patch("umpost.um2netcdf.fix_level_coord"), mock.patch("umpost.um2netcdf.apply_mask"), # TODO: eventually call real version mock.patch("umpost.um2netcdf.cubewrite"), @@ -249,7 +249,7 @@ def test_process_no_masking_keep_all_cubes(air_temp_cube, precipitation_flux_cub mock.patch("iris.load") as m_iris_load, mock.patch("iris.fileformats.netcdf.Saver") as m_saver, # prevent I/O - mock.patch("umpost.um2netcdf.fix_latlon_coord"), + mock.patch("umpost.um2netcdf.fix_latlon_coords"), mock.patch("umpost.um2netcdf.fix_level_coord"), mock.patch("umpost.um2netcdf.cubewrite"), ): @@ -396,7 +396,10 @@ def __init__(self, item_code, var_name=None, attributes=None, units=None): self.units = None or units self.standard_name = None self.long_name = None - self.coord = {} + # TODO: Can I remove this... It breaks DummyCubeWithCoords + # It could be required if apply_mask is called during process + # tests. Would cause a KeyError anyway? + # self.coord = {} def name(self): # mimic iris API @@ -756,23 +759,23 @@ def lon_points_standard_dlon_EG(): def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): - assert len(lat_river_points) == um2nc.NUM_LAT_RIVER_GRID + assert len(lat_river_points) == um2nc.NUM_LAT_RIVER_GRID_POINTS assert um2nc.is_lat_river_grid(lat_river_points) # latitude points on ESM1.5 N96 not_lat_river_points = lat_points_standard_dlat_ND[0] - assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID + assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID_POINTS assert not um2nc.is_lat_river_grid(not_lat_river_points) def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): - assert len(lon_river_points) == um2nc.NUM_LON_RIVER_GRID + assert len(lon_river_points) == um2nc.NUM_LON_RIVER_GRID_POINTS assert um2nc.is_lon_river_grid(lon_river_points) # longitude points on normal ESM1.5 N96 grid not_lon_river_points = lon_points_standard_dlon_ND[0] - assert len(not_lon_river_points) != um2nc.NUM_LON_RIVER_GRID + assert len(not_lon_river_points) != um2nc.NUM_LON_RIVER_GRID_POINTS assert not um2nc.is_lon_river_grid(not_lon_river_points) @@ -1000,20 +1003,20 @@ def test_fix_lon_coord_name_error(coordinate_fake_name): class DummyCubeWithCoords(DummyCube): - # DummyCube with coordinates, which can be filled with - # DummyCoordinate objects for testing. - def __init__(self, item_code, var_name=None, attributes=None, units=None, coords = {}): + # DummyCube with coordinates, which can be filled with + # DummyCoordinate objects for testing. + def __init__(self, item_code, var_name=None, attributes=None, units=None, coords = {}): super().__init__(item_code, var_name, attributes, units) self.coordinate_dict = coords - def coord(self, coordinate_name): + def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] @pytest.fixture def ua_plev_cube_with_latlon_coords(ua_plev_cube): - lat_points = np.array([-90., -88.75, -87.5 ], dtype = "float32") - lon_points = np.array([ 0., 1.875, 3.75 ], dtype = "float32") + lat_points = np.array([-90., -88.75, -87.5], dtype = "float32") + lon_points = np.array([ 0., 1.875, 3.75], dtype = "float32") lat_coord_object = DummyCoordinate( um2nc.LAT_COORD_NAME, @@ -1030,9 +1033,9 @@ def ua_plev_cube_with_latlon_coords(ua_plev_cube): } cube_with_coords = DummyCubeWithCoords( - ua_plev_cube.item_code, + ua_plev_cube.item_code, ua_plev_cube.var_name, - coords = coords_dict + coords=coords_dict ) return cube_with_coords @@ -1051,14 +1054,13 @@ def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): assert lat_coord.points.dtype == np.dtype("float32") assert lon_coord.points.dtype == np.dtype("float32") - # Mock additional functions called by um2nc.fix_latlon_coords, as they may - # require methods not implemented by the DummyCubeWithCoordinates. + # require methods not implemented by the DummyCubeWithCoordinates. with ( - mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value = None), - mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value = None), - mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value = None) + mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value=None), + mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value=None), + mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value=None) ): um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) @@ -1066,25 +1068,23 @@ def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): assert lon_coord.points.dtype == np.dtype("float64") - def test_fix_latlon_coords_error(ua_plev_cube_with_latlon_coords): # Test that fix_latlon_coords raises the right type of error when a cube # is missing coordinates def _raise_CoordinateNotFoundError(coord_name): - # Include an argument "coord_name" to mimic the signature of an + # Include an argument "coord_name" to mimic the signature of an # Iris cube's ".coord" method - raise CoordinateNotFoundError(coord_name) - + raise iris.exceptions.CoordinateNotFoundError(coord_name) + # Following values don't matter for test. Just needed as arguments grid_type = um2nc.GRID_NEW_DYNAMICS dlat = 1.25 dlon = 1.875 - - # Replace coord method to raise CoordinateNotFoundError + + # Replace coord method to raise UnsupportedTimeSeriesError ua_plev_cube_with_latlon_coords.coord = _raise_CoordinateNotFoundError with ( - pytest.warns(), - pytest.raises(CoordinateNotFoundError) + pytest.raises(um2nc.UnsupportedTimeSeriesError) ): um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index b395e2b..5dec69a 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -49,9 +49,9 @@ LAT_COORD_NAME: np.array([[-90., 90.]]) } -NUM_LAT_RIVER_GRID = 180 -NUM_LON_RIVER_GRID = 360 - +NUM_LAT_RIVER_GRID_POINTS = 180 +NUM_LON_RIVER_GRID_POINTS = 360 + NC_FORMATS = { 1: 'NETCDF3_CLASSIC', @@ -66,6 +66,14 @@ class PostProcessingError(Exception): pass +class UnsupportedTimeSeriesError(PostProcessingError): + """ + Error to be raised when latitude and longitude coordinates + are missing. + """ + pass + + # Override the PP file calendar function to use Proleptic Gregorian rather than Gregorian. # This matters for control runs with model years < 1600. @property @@ -117,13 +125,14 @@ def fix_lat_coord_name(lat_coordinate, grid_type, dlat): Add a 'var_name' attribute to a latitude coordinate object based on the grid it lies on. + NB - Grid spacing dlon only refers to variables on the main + horizontal grids, and not the river grid. + Parameters ---------- lat_coordinate: coordinate object from iris cube (edits in place). grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. - NB - Only applies to variables on the main horizontal grids, - and not the river grid. """ if lat_coordinate.name() != LAT_COORD_NAME: @@ -139,18 +148,20 @@ def fix_lat_coord_name(lat_coordinate, grid_type, dlat): else: lat_coordinate.var_name = 'lat' + def fix_lon_coord_name(lon_coordinate, grid_type, dlon): """ Add a 'var_name' attribute to a longitude coordinate object based on the grid it lies on. + NB - Grid spacing dlon only refers to variables on the main + horizontal grids, and not the river grid. + Parameters ---------- lon_coordinate: coordinate object from iris cube (edits in place). grid_type: (string) model horizontal grid type. dlon: (float) zonal spacing between longitude grid points. - NB - Only applies to variables on the main horizontal grids, - and not the river grid. """ if lon_coordinate.name() != LON_COORD_NAME: @@ -158,7 +169,7 @@ def fix_lon_coord_name(lon_coordinate, grid_type, dlon): f"Wrong coordinate {lon_coordinate.name()} supplied. " f"Expected {LAT_COORD_NAME}." ) - + if is_lon_river_grid(lon_coordinate.points): lon_coordinate.var_name = 'lon_river' elif is_lon_u_grid(lon_coordinate.points, grid_type, dlon): @@ -175,7 +186,7 @@ def is_lat_river_grid(latitude_points): ---------- latitude_points: (array) 1D array of latitude grid points. """ - return len(latitude_points) == NUM_LAT_RIVER_GRID + return len(latitude_points) == NUM_LAT_RIVER_GRID_POINTS def is_lon_river_grid(longitude_points): @@ -187,7 +198,7 @@ def is_lon_river_grid(longitude_points): longitude_points: (array) 1D array of longitude grid points. """ - return len(longitude_points) == NUM_LON_RIVER_GRID + return len(longitude_points) == NUM_LON_RIVER_GRID_POINTS def is_lat_v_grid(latitude_points, grid_type, dlat): @@ -200,17 +211,17 @@ def is_lat_v_grid(latitude_points, grid_type, dlat): grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. """ - + min_latitude = latitude_points[0] min_lat_v_nd_grid = -90.+0.5*dlat min_lat_v_eg_grid = -90 - return ( - (min_latitude == min_lat_v_eg_grid and grid_type == GRID_END_GAME) or - ( - np.allclose(min_lat_v_nd_grid, min_latitude) and - grid_type == GRID_NEW_DYNAMICS - ) - ) + + if grid_type == GRID_END_GAME: + return min_latitude == min_lat_v_eg_grid + elif grid_type == GRID_NEW_DYNAMICS: + return np.allclose(min_lat_v_nd_grid, min_latitude) + else: + return False # TODO: Is this situation ever valid? def is_lon_u_grid(longitude_points, grid_type, dlon): @@ -227,13 +238,12 @@ def is_lon_u_grid(longitude_points, grid_type, dlon): min_lon_u_nd_grid = 0.5*dlon min_lon_u_eg_grid = 0 - return ( - (min_longitude == min_lon_u_eg_grid and grid_type == GRID_END_GAME) or - ( - np.allclose(min_lon_u_nd_grid, min_longitude) and - grid_type == GRID_NEW_DYNAMICS - ) - ) + if grid_type == GRID_END_GAME: + return min_longitude == min_lon_u_eg_grid + elif grid_type == GRID_NEW_DYNAMICS: + return np.allclose(min_lon_u_nd_grid, min_longitude) + else: + return False # TODO: Is this situation ever valid? def add_latlon_coord_bounds(cube_coordinate): @@ -257,7 +267,8 @@ def add_latlon_coord_bounds(cube_coordinate): if len(cube_coordinate.points) > 1: cube_coordinate.guess_bounds() else: - # For length 1, assume it's global. guess_bounds doesn't work in this case. + # For length 1, assume it's global. + # guess_bounds doesn't work in this case. cube_coordinate.bounds = GLOBAL_COORD_BOUNDS[coordinate_name] @@ -266,8 +277,8 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): Wrapper function to modify cube's horizontal coordinates (latitude and longitude). Converts to float64, adds grid bounds, and renames coordinates. Modifies cube in place. - - NB - grid spacings dlat and dlon only refer to variables on the main + + NB - grid spacings dlat and dlon only refer to variables on the main horizontal grids, and not the river grid. Parameters @@ -275,7 +286,7 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): cube: Iris cube object (modified in place). grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. - dlon: (float) zonal spacing between longitude grid points. + dlon: (float) zonal spacing between longitude grid points. """ # TODO: Check that we don't need the double coordinate lookup. @@ -283,23 +294,21 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): latitude_coordinate = cube.coord(LAT_COORD_NAME) longitude_coordinate = cube.coord(LON_COORD_NAME) except iris.exceptions.CoordinateNotFoundError: - warnings.warn( + msg = ( "Missing latitude or longitude coordinate for variable (possible timeseries?): \n" f"{cube}\n" ) - raise - + raise UnsupportedTimeSeriesError(msg) + # Force to double for consistency with CMOR latitude_coordinate.points = latitude_coordinate.points.astype(np.float64) longitude_coordinate.points = longitude_coordinate.points.astype(np.float64) - + add_latlon_coord_bounds(latitude_coordinate) add_latlon_coord_bounds(longitude_coordinate) - + fix_lat_coord_name(latitude_coordinate, grid_type, dlat) fix_lon_coord_name(longitude_coordinate, grid_type, dlon) - - # TODO: refactor to "rename level coord" From 8e458dc680a639e262e4bd0ec604c93e1fe6adfb Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Wed, 14 Aug 2024 23:09:19 +1000 Subject: [PATCH 12/27] Finally installed a linter... --- test/test_um2netcdf.py | 127 +++++++++++++++++++++-------------------- umpost/um2netcdf.py | 8 +-- 2 files changed, 68 insertions(+), 67 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index fec6ac3..32c95a0 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -692,17 +692,17 @@ def lon_u_points_dlon_ND(): @pytest.fixture def lat_v_points_dlat_EG(): - # Array of latitude points and corresponding spacing imitating + # Array of latitude points and corresponding spacing imitating # the real lat_v grid from CM2 which uses grid type EG. - dlat = 1.25 + dlat = 1.25 lat_v_points = np.arange(-90., 91, dlat) return (lat_v_points, um2nc.GRID_END_GAME, dlat) @pytest.fixture def lon_u_points_dlon_EG(): - # Array of longitude points and corresponding spacing imitating + # Array of longitude points and corresponding spacing imitating # the real lon_u grid from CM2 which uses grid type EG. dlon = 1.875 @@ -712,49 +712,49 @@ def lon_u_points_dlon_EG(): @pytest.fixture def lat_points_standard_dlat_ND(): - # Array of latitude points and corresponding spacing imitating the - # standard (not river or v) lat grid for ESM1.5 which uses + # Array of latitude points and corresponding spacing imitating the + # standard (not river or v) lat grid for ESM1.5 which uses # grid type ND. dlat = 1.25 - lat_points_middle = np.arange(-88.75, 89., 1.25) + lat_points_middle = np.arange(-88.75, 89., 1.25) lat_points = np.concatenate(([-90], lat_points_middle, [90.] - )) + )) return (lat_points, um2nc.GRID_NEW_DYNAMICS, dlat) @pytest.fixture def lon_points_standard_dlon_ND(): - # Array of longitude points and corresponding spacing imitating the - # standard (not river or u) lon grid for ESM1.5 which uses grid + # Array of longitude points and corresponding spacing imitating the + # standard (not river or u) lon grid for ESM1.5 which uses grid # type ND. dlon = 1.875 - lon_points = np.arange(0, 360, dlon) + lon_points = np.arange(0, 360, dlon) return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) @pytest.fixture def lat_points_standard_dlat_EG(): - # Array of latitude points and corresponding spacing imitating the - # standard (not river or v) lat grid for CM2 which uses + # Array of latitude points and corresponding spacing imitating the + # standard (not river or v) lat grid for CM2 which uses # grid type EG. dlat = 1.25 - lat_points = np.arange(-90 + 0.5*dlat, 90., dlat) + lat_points = np.arange(-90 + 0.5*dlat, 90., dlat) return (lat_points, um2nc.GRID_END_GAME, dlat) @pytest.fixture def lon_points_standard_dlon_EG(): - # Array of longitude points and corresponding spacing imitating the - # standard (not river or u) lon grid for CM2 which uses grid + # Array of longitude points and corresponding spacing imitating the + # standard (not river or u) lon grid for CM2 which uses grid # type EG. dlon = 1.875 - lon_points = np.arange(0.5*dlon, 360, dlon) + lon_points = np.arange(0.5*dlon, 360, dlon) return (lon_points, um2nc.GRID_END_GAME, dlon) @@ -764,7 +764,7 @@ def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): # latitude points on ESM1.5 N96 not_lat_river_points = lat_points_standard_dlat_ND[0] - + assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID_POINTS assert not um2nc.is_lat_river_grid(not_lat_river_points) @@ -779,15 +779,15 @@ def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): assert not um2nc.is_lon_river_grid(not_lon_river_points) -def test_is_lat_v_grid(lat_v_points_dlat_EG, +def test_is_lat_v_grid(lat_v_points_dlat_EG, lat_v_points_dlat_ND, lat_points_standard_dlat_ND, lat_points_standard_dlat_EG - ): - lat_v_points, grid_code, dlat = lat_v_points_dlat_EG + ): + lat_v_points, grid_code, dlat = lat_v_points_dlat_EG assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) - lat_v_points, grid_code, dlat = lat_v_points_dlat_ND + lat_v_points, grid_code, dlat = lat_v_points_dlat_ND assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_EG @@ -797,15 +797,15 @@ def test_is_lat_v_grid(lat_v_points_dlat_EG, assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) -def test_is_lon_u_grid(lon_u_points_dlon_EG, +def test_is_lon_u_grid(lon_u_points_dlon_EG, lon_u_points_dlon_ND, lon_points_standard_dlon_ND, - lon_points_standard_dlon_EG - ): - lon_u_points, grid_code, dlon = lon_u_points_dlon_EG + lon_points_standard_dlon_EG + ): + lon_u_points, grid_code, dlon = lon_u_points_dlon_EG assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) - lon_u_points, grid_code, dlon = lon_u_points_dlon_ND + lon_u_points, grid_code, dlon = lon_u_points_dlon_ND assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_EG @@ -825,10 +825,11 @@ class DummyCoordinate: bounds: np.ndarray = None # Note that var_name attribute is different thing to return # value of name() method. - var_name: str = None - + var_name: str = None + def name(self): return self.coordname + def has_bounds(self): return self.bounds is not None @@ -836,22 +837,22 @@ def has_bounds(self): def test_add_latlon_coord_bounds_has_bounds(): # Test that bounds are not modified if they already exist lon_points = np.array([1., 2., 3.]) - lon_bounds = np.array([[0.5, 1.5], - [1.5, 2.5], - [2.5, 3.5]]) + lon_bounds = np.array([[0.5, 1.5], + [1.5, 2.5], + [2.5, 3.5]]) lon_coord_with_bounds = DummyCoordinate( um2nc.LON_COORD_NAME, lon_points, lon_bounds ) assert lon_coord_with_bounds.has_bounds() - + um2nc.add_latlon_coord_bounds(lon_coord_with_bounds) assert np.array_equal(lon_coord_with_bounds.bounds, lon_bounds) def test_add_latlon_coord_guess_bounds(): - # Test that guess_bounds method is called when + # Test that guess_bounds method is called when # coordinate has no bounds and length > 1. lon_points = np.array([0., 1.]) lon_coord_nobounds = DummyCoordinate( @@ -861,7 +862,7 @@ def test_add_latlon_coord_guess_bounds(): # Mock Iris' guess_bounds method to check whether it is called lon_coord_nobounds.guess_bounds = mock.Mock(return_value=None) - + assert len(lon_coord_nobounds.points) > 1 assert not lon_coord_nobounds.has_bounds() @@ -872,7 +873,7 @@ def test_add_latlon_coord_guess_bounds(): def test_add_latlon_coord_single(): # Test that the correct global bounds are added to coordinates - # with just a single point. + # with just a single point. for coord_name in [um2nc.LON_COORD_NAME, um2nc.LAT_COORD_NAME]: points = np.array([0.]) coord_single_point = DummyCoordinate( @@ -906,32 +907,32 @@ def test_fix_lat_coord_name(): # Following values are ignored due to mocking of checking functions. grid_type = um2nc.GRID_END_GAME dlat = 1.875 - lat_points = np.array([1.,2.,3.]) + lat_points = np.array([1., 2., 3.]) latitude_coordinate = DummyCoordinate( um2nc.LAT_COORD_NAME, lat_points ) - assert latitude_coordinate.var_name is None + assert latitude_coordinate.var_name is None - # Mock the return value of grid checking functions in order to simplify test setup, + # Mock the return value of grid checking functions in order to simplify test setup, # since grid checking functions have their own tests. - with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = True): + with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=True): um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) assert latitude_coordinate.var_name == "lat_river" - latitude_coordinate.var_name = None + latitude_coordinate.var_name = None with ( - mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = False), - mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value = True) + mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), + mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=True) ): um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) assert latitude_coordinate.var_name == "lat_v" - latitude_coordinate.var_name = None + latitude_coordinate.var_name = None with ( - mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value = False), - mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value = False) + mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), + mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=False) ): um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) assert latitude_coordinate.var_name == "lat" @@ -941,32 +942,32 @@ def test_fix_lon_coord_name(): # Following values are ignored due to mocking of checking functions. grid_type = um2nc.GRID_END_GAME dlon = 1.875 - lon_points = np.array([1.,2.,3.]) + lon_points = np.array([1., 2., 3.]) longitude_coordinate = DummyCoordinate( um2nc.LON_COORD_NAME, lon_points ) - assert longitude_coordinate.var_name is None + assert longitude_coordinate.var_name is None - # Mock the return value of grid checking functions in order to simplify test setup, + # Mock the return value of grid checking functions in order to simplify test setup, # since grid checking functions have their own tests. - with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = True): + with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=True): um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) assert longitude_coordinate.var_name == "lon_river" longitude_coordinate.var_name = None with ( - mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = False), - mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value = True) + mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), + mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=True) ): um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) assert longitude_coordinate.var_name == "lon_u" - longitude_coordinate.var_name = None + longitude_coordinate.var_name = None with ( - mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value = False), - mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value = False) + mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), + mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=False) ): um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) assert longitude_coordinate.var_name == "lon" @@ -1003,20 +1004,20 @@ def test_fix_lon_coord_name_error(coordinate_fake_name): class DummyCubeWithCoords(DummyCube): - # DummyCube with coordinates, which can be filled with + # DummyCube with coordinates, which can be filled with # DummyCoordinate objects for testing. - def __init__(self, item_code, var_name=None, attributes=None, units=None, coords = {}): + def __init__(self, item_code, var_name=None, attributes=None, units=None, coords={}): super().__init__(item_code, var_name, attributes, units) - self.coordinate_dict = coords + self.coordinate_dict = coords def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] - + @pytest.fixture def ua_plev_cube_with_latlon_coords(ua_plev_cube): - lat_points = np.array([-90., -88.75, -87.5], dtype = "float32") - lon_points = np.array([ 0., 1.875, 3.75], dtype = "float32") + lat_points = np.array([-90., -88.75, -87.5], dtype="float32") + lon_points = np.array([0., 1.875, 3.75], dtype="float32") lat_coord_object = DummyCoordinate( um2nc.LAT_COORD_NAME, @@ -1036,14 +1037,14 @@ def ua_plev_cube_with_latlon_coords(ua_plev_cube): ua_plev_cube.item_code, ua_plev_cube.var_name, coords=coords_dict - ) - + ) + return cube_with_coords def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): # Test that coordinate arrays are converted to float64 - + # Following values don't matter for test. Just needed as arguments grid_type = um2nc.GRID_NEW_DYNAMICS dlat = 1.25 diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 5dec69a..61b5420 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -68,7 +68,7 @@ class PostProcessingError(Exception): class UnsupportedTimeSeriesError(PostProcessingError): """ - Error to be raised when latitude and longitude coordinates + Error to be raised when latitude and longitude coordinates are missing. """ pass @@ -161,7 +161,7 @@ def fix_lon_coord_name(lon_coordinate, grid_type, dlon): ---------- lon_coordinate: coordinate object from iris cube (edits in place). grid_type: (string) model horizontal grid type. - dlon: (float) zonal spacing between longitude grid points. + dlon: (float) zonal spacing between longitude grid points. """ if lon_coordinate.name() != LON_COORD_NAME: @@ -232,7 +232,7 @@ def is_lon_u_grid(longitude_points, grid_type, dlon): ---------- longitude_points: (array) 1D array of longitude grid points. grid_type: (string) model horizontal grid type. - dlon: (float) zonal spacing between longitude grid points. + dlon: (float) zonal spacing between longitude grid points. """ min_longitude = longitude_points[0] min_lon_u_nd_grid = 0.5*dlon @@ -274,7 +274,7 @@ def add_latlon_coord_bounds(cube_coordinate): def fix_latlon_coords(cube, grid_type, dlat, dlon): """ - Wrapper function to modify cube's horizontal coordinates + Wrapper function to modify cube's horizontal coordinates (latitude and longitude). Converts to float64, adds grid bounds, and renames coordinates. Modifies cube in place. From 00f503c4386f254ebefc476fca46e26c11088326 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Thu, 15 Aug 2024 13:35:42 +1000 Subject: [PATCH 13/27] Add end to end test of fix_latlon_coords and required dummy coordinate features --- test/test_um2netcdf.py | 817 +++++++++++++++++++++++++---------------- umpost/um2netcdf.py | 1 - 2 files changed, 495 insertions(+), 323 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 32c95a0..94d00ac 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -56,7 +56,10 @@ def z_sea_theta_data(): @pytest.fixture def mule_vars(z_sea_rho_data, z_sea_theta_data): - """Simulate mule variables from aiihca.paa1jan.subset data.""" + """ + Simulate mule variables for the New Dynamics grid from + aiihca.paa1jan.subset data. + """ d_lat = 1.25 # spacing manually copied from aiihca.paa1jan.subset file d_lon = 1.875 return um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, z_sea_rho_data, z_sea_theta_data) @@ -655,437 +658,607 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units -@pytest.fixture -def lat_river_points(): - # Array of points imitating real latitudes on UM V7.3's river grid. - # Must have length 180. - return np.arange(-90., 90) + 0.5 - - -@pytest.fixture -def lon_river_points(): - # Array of points imitating real longitudes on UM V7.3's river grid. - # Must have length 360. - return np.arange(0., 360.) + 0.5 - - -@pytest.fixture -def lat_v_points_dlat_ND(): - # Array of latitude points and corresponding spacing imitating the real - # lat_v grid from ESM1.5 (which uses the New Dynamics grid). - - dlat = 1.25 - lat_v_points = np.arange(-90.+0.5*dlat, 90, dlat) - return (lat_v_points, um2nc.GRID_NEW_DYNAMICS, dlat) - +# @pytest.fixture +# def lat_v_points_dlat_EG(): +# # Array of latitude points and corresponding spacing imitating +# # the real lat_v grid from CM2 which uses grid type EG. -@pytest.fixture -def lon_u_points_dlon_ND(): - # Array of latitude points and corresponding spacing imitating the real - # lon_u grid from ESM1.5 (which uses the New Dynamics grid). +# dlat = 1.25 +# lat_v_points = np.arange(-90., 91, dlat) +# return (lat_v_points, um2nc.GRID_END_GAME, dlat) - dlon = 1.875 - lon_u_points = np.arange(0.5*dlon, 360, dlon) - return (lon_u_points, um2nc.GRID_NEW_DYNAMICS, dlon) +# @pytest.fixture +# def lon_u_points_dlon_EG(): +# # Array of longitude points and corresponding spacing imitating +# # the real lon_u grid from CM2 which uses grid type EG. +# dlon = 1.875 +# lon_u_points = np.arange(0, 360, dlon) +# return (lon_u_points, um2nc.GRID_END_GAME, dlon) -@pytest.fixture -def lat_v_points_dlat_EG(): - # Array of latitude points and corresponding spacing imitating - # the real lat_v grid from CM2 which uses grid type EG. - dlat = 1.25 - lat_v_points = np.arange(-90., 91, dlat) - return (lat_v_points, um2nc.GRID_END_GAME, dlat) +# @pytest.fixture +# def lat_points_standard_dlat_ND(): +# # Array of latitude points and corresponding spacing imitating the +# # standard (not river or v) lat grid for ESM1.5 which uses +# # grid type ND. +# dlat = 1.25 +# lat_points_middle = np.arange(-88.75, 89., 1.25) +# lat_points = np.concatenate(([-90], +# lat_points_middle, +# [90.] +# )) +# return (lat_points, um2nc.GRID_NEW_DYNAMICS, dlat) -@pytest.fixture -def lon_u_points_dlon_EG(): - # Array of longitude points and corresponding spacing imitating - # the real lon_u grid from CM2 which uses grid type EG. - dlon = 1.875 - lon_u_points = np.arange(0, 360, dlon) - return (lon_u_points, um2nc.GRID_END_GAME, dlon) +# @pytest.fixture +# def lon_points_standard_dlon_ND(): +# # Array of longitude points and corresponding spacing imitating the +# # standard (not river or u) lon grid for ESM1.5 which uses grid +# # type ND. +# dlon = 1.875 +# lon_points = np.arange(0, 360, dlon) +# return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) -@pytest.fixture -def lat_points_standard_dlat_ND(): - # Array of latitude points and corresponding spacing imitating the - # standard (not river or v) lat grid for ESM1.5 which uses - # grid type ND. - dlat = 1.25 - lat_points_middle = np.arange(-88.75, 89., 1.25) - lat_points = np.concatenate(([-90], - lat_points_middle, - [90.] - )) - return (lat_points, um2nc.GRID_NEW_DYNAMICS, dlat) +# @pytest.fixture +# def lat_points_standard_dlat_EG(): +# # Array of latitude points and corresponding spacing imitating the +# # standard (not river or v) lat grid for CM2 which uses +# # grid type EG. +# dlat = 1.25 +# lat_points = np.arange(-90 + 0.5*dlat, 90., dlat) +# return (lat_points, um2nc.GRID_END_GAME, dlat) -@pytest.fixture -def lon_points_standard_dlon_ND(): - # Array of longitude points and corresponding spacing imitating the - # standard (not river or u) lon grid for ESM1.5 which uses grid - # type ND. - - dlon = 1.875 - lon_points = np.arange(0, 360, dlon) - return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) - - -@pytest.fixture -def lat_points_standard_dlat_EG(): - # Array of latitude points and corresponding spacing imitating the - # standard (not river or v) lat grid for CM2 which uses - # grid type EG. - dlat = 1.25 - lat_points = np.arange(-90 + 0.5*dlat, 90., dlat) - return (lat_points, um2nc.GRID_END_GAME, dlat) +# @pytest.fixture +# def lon_points_standard_dlon_EG(): +# # Array of longitude points and corresponding spacing imitating the +# # standard (not river or u) lon grid for CM2 which uses grid +# # type EG. +# dlon = 1.875 +# lon_points = np.arange(0.5*dlon, 360, dlon) +# return (lon_points, um2nc.GRID_END_GAME, dlon) -@pytest.fixture -def lon_points_standard_dlon_EG(): - # Array of longitude points and corresponding spacing imitating the - # standard (not river or u) lon grid for CM2 which uses grid - # type EG. - dlon = 1.875 - lon_points = np.arange(0.5*dlon, 360, dlon) - return (lon_points, um2nc.GRID_END_GAME, dlon) +# def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): +# assert len(lat_river_points) == um2nc.NUM_LAT_RIVER_GRID_POINTS +# assert um2nc.is_lat_river_grid(lat_river_points) +# # latitude points on ESM1.5 N96 +# not_lat_river_points = lat_points_standard_dlat_ND[0] -def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): - assert len(lat_river_points) == um2nc.NUM_LAT_RIVER_GRID_POINTS - assert um2nc.is_lat_river_grid(lat_river_points) +# assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID_POINTS +# assert not um2nc.is_lat_river_grid(not_lat_river_points) - # latitude points on ESM1.5 N96 - not_lat_river_points = lat_points_standard_dlat_ND[0] - assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID_POINTS - assert not um2nc.is_lat_river_grid(not_lat_river_points) +# def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): +# assert len(lon_river_points) == um2nc.NUM_LON_RIVER_GRID_POINTS +# assert um2nc.is_lon_river_grid(lon_river_points) +# # longitude points on normal ESM1.5 N96 grid +# not_lon_river_points = lon_points_standard_dlon_ND[0] +# assert len(not_lon_river_points) != um2nc.NUM_LON_RIVER_GRID_POINTS +# assert not um2nc.is_lon_river_grid(not_lon_river_points) -def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): - assert len(lon_river_points) == um2nc.NUM_LON_RIVER_GRID_POINTS - assert um2nc.is_lon_river_grid(lon_river_points) - # longitude points on normal ESM1.5 N96 grid - not_lon_river_points = lon_points_standard_dlon_ND[0] - assert len(not_lon_river_points) != um2nc.NUM_LON_RIVER_GRID_POINTS - assert not um2nc.is_lon_river_grid(not_lon_river_points) +# def test_is_lat_v_grid(lat_v_points_dlat_EG, +# lat_v_points_dlat_ND, +# lat_points_standard_dlat_ND, +# lat_points_standard_dlat_EG +# ): +# lat_v_points, grid_code, dlat = lat_v_points_dlat_EG +# assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) +# lat_v_points, grid_code, dlat = lat_v_points_dlat_ND +# assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) -def test_is_lat_v_grid(lat_v_points_dlat_EG, - lat_v_points_dlat_ND, - lat_points_standard_dlat_ND, - lat_points_standard_dlat_EG - ): - lat_v_points, grid_code, dlat = lat_v_points_dlat_EG - assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) +# not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_EG +# assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) - lat_v_points, grid_code, dlat = lat_v_points_dlat_ND - assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) +# not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_ND +# assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) - not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_EG - assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) - not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_ND - assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) +# def test_is_lon_u_grid(lon_u_points_dlon_EG, +# lon_u_points_dlon_ND, +# lon_points_standard_dlon_ND, +# lon_points_standard_dlon_EG +# ): +# lon_u_points, grid_code, dlon = lon_u_points_dlon_EG +# assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) +# lon_u_points, grid_code, dlon = lon_u_points_dlon_ND +# assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) -def test_is_lon_u_grid(lon_u_points_dlon_EG, - lon_u_points_dlon_ND, - lon_points_standard_dlon_ND, - lon_points_standard_dlon_EG - ): - lon_u_points, grid_code, dlon = lon_u_points_dlon_EG - assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) +# not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_EG +# assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) - lon_u_points, grid_code, dlon = lon_u_points_dlon_ND - assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) +# not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_ND +# assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) - not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_EG - assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) +# Completely faked bounds to be added to a DummyCoordinate when +# its guess_bounds method is called. - not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_ND - assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) +DUMMY_GUESSED_BOUNDS = "dummy_guessed_bounds" @dataclass class DummyCoordinate: """ Imitation cube coordinate for unit testing. + Includes additional information that is not part of + a standard iris cube coordinate, but is useful for testing, + including MuleVars grid metadata, and atribute values expected + after testing. """ coordname: str points: np.ndarray - bounds: np.ndarray = None + # NB: MuleVars is not part of an iris coordinate, but is packaged + # in here to aid with testing. + mulevars: um2nc.MuleVars + + bounds = None # Note that var_name attribute is different thing to return # value of name() method. var_name: str = None + # Expected attributes after testing: + expected_var_name: str = None + expected_points_type: str = None + expected_bounds: any = None + def name(self): return self.coordname def has_bounds(self): return self.bounds is not None + def guess_bounds(self): + self.bounds = DUMMY_GUESSED_BOUNDS -def test_add_latlon_coord_bounds_has_bounds(): - # Test that bounds are not modified if they already exist - lon_points = np.array([1., 2., 3.]) - lon_bounds = np.array([[0.5, 1.5], - [1.5, 2.5], - [2.5, 3.5]]) - lon_coord_with_bounds = DummyCoordinate( - um2nc.LON_COORD_NAME, - lon_points, - lon_bounds +@pytest.fixture +def lat_river_dummy(): + # Dummy coordinate imitating UM V7.3s river grid. Must have length 180. + lat_river_points = np.arange(-90., 90, dtype="float32") + 0.5 + + d_lat = 1.25 + d_lon = 1.875 + mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) + + lat_river_dummy_coord = DummyCoordinate( + coordname=um2nc.LAT_COORD_NAME, + points=lat_river_points, + mulevars=mv, + # Expected values after fix_latlon_tests + expected_var_name="lat_river", + expected_points_type=np.dtype("float64"), + expected_bounds=DUMMY_GUESSED_BOUNDS ) - assert lon_coord_with_bounds.has_bounds() - um2nc.add_latlon_coord_bounds(lon_coord_with_bounds) - assert np.array_equal(lon_coord_with_bounds.bounds, lon_bounds) + return lat_river_dummy_coord +@pytest.fixture +def lon_river_dummy(): + # Dummy coordinate imitating UM V7.3s river grid. Must have length 90. + lon_river_points = np.arange(0., 360., dtype="float32") + 0.5 -def test_add_latlon_coord_guess_bounds(): - # Test that guess_bounds method is called when - # coordinate has no bounds and length > 1. - lon_points = np.array([0., 1.]) - lon_coord_nobounds = DummyCoordinate( - um2nc.LON_COORD_NAME, - lon_points + d_lat = 1.25 + d_lon = 1.875 + mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) + + lon_river_dummy_coord = DummyCoordinate( + coordname=um2nc.LON_COORD_NAME, + points=lon_river_points, + mulevars=mv, + # Expected values after fix_latlon_tests + expected_var_name="lon_river", + expected_points_type=np.dtype("float64"), + expected_bounds=DUMMY_GUESSED_BOUNDS ) - # Mock Iris' guess_bounds method to check whether it is called - lon_coord_nobounds.guess_bounds = mock.Mock(return_value=None) + return lon_river_dummy_coord - assert len(lon_coord_nobounds.points) > 1 - assert not lon_coord_nobounds.has_bounds() - um2nc.add_latlon_coord_bounds(lon_coord_nobounds) +@pytest.fixture +def lat_v_nd_dummy(): + # Dummy coordinate imitating imitating the real + # lat_v grid from ESM1.5 (which uses the New Dynamics grid). - lon_coord_nobounds.guess_bounds.assert_called() + d_lat = 1.25 + d_lon = 1.875 + lat_v_points = np.arange(-90.+0.5*d_lat, 90, d_lat, dtype="float32") + + mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) + + lat_v_nd_dummy_coord = DummyCoordinate( + coordname=um2nc.LAT_COORD_NAME, + points=lat_v_points, + mulevars=mv, + # Expected values after fix_latlon_tests + expected_var_name="lat_v", + expected_points_type=np.dtype("float64"), + expected_bounds=DUMMY_GUESSED_BOUNDS + ) + return lat_v_nd_dummy_coord -def test_add_latlon_coord_single(): - # Test that the correct global bounds are added to coordinates - # with just a single point. - for coord_name in [um2nc.LON_COORD_NAME, um2nc.LAT_COORD_NAME]: - points = np.array([0.]) - coord_single_point = DummyCoordinate( - coord_name, - points - ) - assert len(coord_single_point.points) == 1 - assert not coord_single_point.has_bounds() +@pytest.fixture +def lon_u_nd_dummy(): + # Dummy coordinate imitating imitating the real + # lon_u grid from ESM1.5 (which uses the New Dynamics grid). - um2nc.add_latlon_coord_bounds(coord_single_point) + d_lat = 1.25 + d_lon = 1.875 + lon_u_points = np.arange(0.5*d_lon, 360, d_lon, dtype="float32") + + mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) + + lon_u_nd_dummy_coord = DummyCoordinate( + coordname=um2nc.LON_COORD_NAME, + points=lon_u_points, + mulevars=mv, + # Expected values after fix_latlon_tests + expected_var_name="lon_u", + expected_points_type=np.dtype("float64"), + expected_bounds=DUMMY_GUESSED_BOUNDS + ) - expected_bounds = um2nc.GLOBAL_COORD_BOUNDS[coord_name] - assert np.array_equal(coord_single_point.bounds, expected_bounds) + return lon_u_nd_dummy_coord -def test_add_latlon_coord_error(): - fake_coord_name = "fake coordinate" - fake_points = np.array([1., 2., 3.]) +class DummyCubeWithCoords(DummyCube): + # DummyCube with coordinates, which can be filled with + # DummyCoordinate objects for testing. + def __init__(self, item_code, var_name=None, attributes=None, units=None, coords={}): + super().__init__(item_code, var_name, attributes, units) + self.coordinate_dict = coords - fake_coord = DummyCoordinate( - fake_coord_name, - fake_points - ) + def coord(self, coordinate_name): + return self.coordinate_dict[coordinate_name] - with pytest.raises(ValueError): - um2nc.add_latlon_coord_bounds(fake_coord) +def test_fix_latlon_coords(ua_plev_cube, + lat_river_dummy, + lon_river_dummy, + lat_v_nd_dummy, + lon_u_nd_dummy): + """ + Tests of the fix_lat_lon_coords function. + fix_latlon_coords makes the following modifications in place: + - Converts coordinate points to double. + - Adds bounds to its coordinates. + - Adds var_name attributes to the coordinates. -def test_fix_lat_coord_name(): - # Following values are ignored due to mocking of checking functions. - grid_type = um2nc.GRID_END_GAME - dlat = 1.875 - lat_points = np.array([1., 2., 3.]) + Here we test that these three things are done correctly. + """ + dummy_coord_pairs = [ + (lat_river_dummy, lon_river_dummy), + (lat_v_nd_dummy, lon_u_nd_dummy) + ] + for lat_coord, lon_coord in dummy_coord_pairs: + + # Check that we're providing a consistent coordinate pair. + assert lat_coord.mulevars == lon_coord.mulevars + + mv = lat_coord.mulevars + + # Construct a dummy cube object with the specified dummy coordinates. + cube_with_coords = DummyCubeWithCoords( + item_code=ua_plev_cube.item_code, + var_name=ua_plev_cube.var_name, + coords={ + um2nc.LAT_COORD_NAME: lat_coord, + um2nc.LON_COORD_NAME: lon_coord + } + ) - latitude_coordinate = DummyCoordinate( - um2nc.LAT_COORD_NAME, - lat_points - ) - assert latitude_coordinate.var_name is None + # Checks prior to modifications. + assert lat_coord.points.dtype == np.dtype("float32") + assert lon_coord.points.dtype == np.dtype("float32") - # Mock the return value of grid checking functions in order to simplify test setup, - # since grid checking functions have their own tests. - with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=True): - um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) - assert latitude_coordinate.var_name == "lat_river" + assert not lat_coord.has_bounds() + assert not lon_coord.has_bounds() - latitude_coordinate.var_name = None - with ( - mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), - mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=True) - ): - um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) - assert latitude_coordinate.var_name == "lat_v" + assert lat_coord.var_name is None + assert lon_coord.var_name is None - latitude_coordinate.var_name = None - with ( - mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), - mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=False) - ): - um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) - assert latitude_coordinate.var_name == "lat" + um2nc.fix_latlon_coords(cube_with_coords, mv.grid_type, mv.d_lat, mv.d_lon) + # Checks post modifications. + assert lat_coord.var_name == lat_coord.expected_var_name + assert lon_coord.var_name == lon_coord.expected_var_name -def test_fix_lon_coord_name(): - # Following values are ignored due to mocking of checking functions. - grid_type = um2nc.GRID_END_GAME - dlon = 1.875 - lon_points = np.array([1., 2., 3.]) + assert lat_coord.points.dtype == lat_coord.expected_points_type + assert lon_coord.points.dtype == lon_coord.expected_points_type - longitude_coordinate = DummyCoordinate( - um2nc.LON_COORD_NAME, - lon_points - ) - assert longitude_coordinate.var_name is None + assert lat_coord.bounds == lat_coord.expected_bounds + assert lon_coord.bounds == lon_coord.expected_bounds - # Mock the return value of grid checking functions in order to simplify test setup, - # since grid checking functions have their own tests. - with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=True): - um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) - assert longitude_coordinate.var_name == "lon_river" - longitude_coordinate.var_name = None - with ( - mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), - mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=True) - ): - um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) - assert longitude_coordinate.var_name == "lon_u" +# def test_add_latlon_coord_bounds_has_bounds(): +# # Test that bounds are not modified if they already exist +# lon_points = np.array([1., 2., 3.]) +# lon_bounds = np.array([[0.5, 1.5], +# [1.5, 2.5], +# [2.5, 3.5]]) +# lon_coord_with_bounds = DummyCoordinate( +# um2nc.LON_COORD_NAME, +# lon_points, +# lon_bounds +# ) +# assert lon_coord_with_bounds.has_bounds() - longitude_coordinate.var_name = None - with ( - mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), - mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=False) - ): - um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) - assert longitude_coordinate.var_name == "lon" +# um2nc.add_latlon_coord_bounds(lon_coord_with_bounds) +# assert np.array_equal(lon_coord_with_bounds.bounds, lon_bounds) -@pytest.fixture -def coordinate_fake_name(): - # Fake dummy coordinate with made up name to test that exceptions are raised - fake_coord_name = "fake coordinate" - fake_points = np.array([1., 2., 3.]) - - fake_coord = DummyCoordinate( - fake_coord_name, - fake_points - ) +# def test_add_latlon_coord_guess_bounds(): +# # Test that guess_bounds method is called when +# # coordinate has no bounds and length > 1. +# lon_points = np.array([0., 1.]) +# lon_coord_nobounds = DummyCoordinate( +# um2nc.LON_COORD_NAME, +# lon_points +# ) + +# # Mock Iris' guess_bounds method to check whether it is called +# lon_coord_nobounds.guess_bounds = mock.Mock(return_value=None) + +# assert len(lon_coord_nobounds.points) > 1 +# assert not lon_coord_nobounds.has_bounds() + +# um2nc.add_latlon_coord_bounds(lon_coord_nobounds) + +# lon_coord_nobounds.guess_bounds.assert_called() + + +# def test_add_latlon_coord_single(): +# # Test that the correct global bounds are added to coordinates +# # with just a single point. +# for coord_name in [um2nc.LON_COORD_NAME, um2nc.LAT_COORD_NAME]: +# points = np.array([0.]) +# coord_single_point = DummyCoordinate( +# coord_name, +# points +# ) + +# assert len(coord_single_point.points) == 1 +# assert not coord_single_point.has_bounds() + +# um2nc.add_latlon_coord_bounds(coord_single_point) + +# expected_bounds = um2nc.GLOBAL_COORD_BOUNDS[coord_name] +# assert np.array_equal(coord_single_point.bounds, expected_bounds) + + +# def test_add_latlon_coord_error(): +# fake_coord_name = "fake coordinate" +# fake_points = np.array([1., 2., 3.]) + +# fake_coord = DummyCoordinate( +# fake_coord_name, +# fake_points +# ) + +# with pytest.raises(ValueError): +# um2nc.add_latlon_coord_bounds(fake_coord) + + +# def test_fix_lat_coord_name(): +# # Following values are ignored due to mocking of checking functions. +# grid_type = um2nc.GRID_END_GAME +# dlat = 1.875 +# lat_points = np.array([1., 2., 3.]) - return fake_coord +# latitude_coordinate = DummyCoordinate( +# um2nc.LAT_COORD_NAME, +# lat_points +# ) +# assert latitude_coordinate.var_name is None + +# # Mock the return value of grid checking functions in order to simplify test setup, +# # since grid checking functions have their own tests. +# with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=True): +# um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) +# assert latitude_coordinate.var_name == "lat_river" + +# latitude_coordinate.var_name = None +# with ( +# mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), +# mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=True) +# ): +# um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) +# assert latitude_coordinate.var_name == "lat_v" + +# latitude_coordinate.var_name = None +# with ( +# mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), +# mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=False) +# ): +# um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) +# assert latitude_coordinate.var_name == "lat" + + +# def test_fix_lon_coord_name(): +# # Following values are ignored due to mocking of checking functions. +# grid_type = um2nc.GRID_END_GAME +# dlon = 1.875 +# lon_points = np.array([1., 2., 3.]) + +# longitude_coordinate = DummyCoordinate( +# um2nc.LON_COORD_NAME, +# lon_points +# ) +# assert longitude_coordinate.var_name is None + +# # Mock the return value of grid checking functions in order to simplify test setup, +# # since grid checking functions have their own tests. +# with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=True): +# um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) +# assert longitude_coordinate.var_name == "lon_river" + +# longitude_coordinate.var_name = None +# with ( +# mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), +# mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=True) +# ): +# um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) +# assert longitude_coordinate.var_name == "lon_u" +# longitude_coordinate.var_name = None +# with ( +# mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), +# mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=False) +# ): +# um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) +# assert longitude_coordinate.var_name == "lon" + + +# @pytest.fixture +# def coordinate_fake_name(): +# # Fake dummy coordinate with made up name to test that exceptions are raised +# fake_coord_name = "fake coordinate" +# fake_points = np.array([1., 2., 3.]) -def test_fix_lat_coord_name_error(coordinate_fake_name): - # Following values unimportant. Just needed as function arguments. - grid_type = um2nc.GRID_NEW_DYNAMICS - dlat = 1.25 - with pytest.raises(ValueError): - um2nc.fix_lat_coord_name(coordinate_fake_name, grid_type, dlat) +# fake_coord = DummyCoordinate( +# fake_coord_name, +# fake_points +# ) +# return fake_coord + + +# def test_fix_lat_coord_name_error(coordinate_fake_name): +# # Following values unimportant. Just needed as function arguments. +# grid_type = um2nc.GRID_NEW_DYNAMICS +# dlat = 1.25 +# with pytest.raises(ValueError): +# um2nc.fix_lat_coord_name(coordinate_fake_name, grid_type, dlat) + + +# def test_fix_lon_coord_name_error(coordinate_fake_name): +# # Following values unimportant. Just needed as function arguments. +# grid_type = um2nc.GRID_NEW_DYNAMICS +# dlon = 1.875 +# with pytest.raises(ValueError): +# um2nc.fix_lon_coord_name(coordinate_fake_name, grid_type, dlon) -def test_fix_lon_coord_name_error(coordinate_fake_name): - # Following values unimportant. Just needed as function arguments. - grid_type = um2nc.GRID_NEW_DYNAMICS - dlon = 1.875 - with pytest.raises(ValueError): - um2nc.fix_lon_coord_name(coordinate_fake_name, grid_type, dlon) -class DummyCubeWithCoords(DummyCube): - # DummyCube with coordinates, which can be filled with - # DummyCoordinate objects for testing. - def __init__(self, item_code, var_name=None, attributes=None, units=None, coords={}): - super().__init__(item_code, var_name, attributes, units) - self.coordinate_dict = coords +# @pytest.fixture +# def ua_plev_cube_with_latlon_coords(ua_plev_cube): +# lat_points = np.array([-90., -88.75, -87.5], dtype="float32") +# lon_points = np.array([0., 1.875, 3.75], dtype="float32") + +# lat_coord_object = DummyCoordinate( +# um2nc.LAT_COORD_NAME, +# lat_points +# ) +# lon_coord_object = DummyCoordinate( +# um2nc.LON_COORD_NAME, +# lon_points +# ) + +# coords_dict = { +# um2nc.LAT_COORD_NAME: lat_coord_object, +# um2nc.LON_COORD_NAME: lon_coord_object +# } + +# cube_with_coords = DummyCubeWithCoords( +# ua_plev_cube.item_code, +# ua_plev_cube.var_name, +# coords=coords_dict +# ) + +# return cube_with_coords + - def coord(self, coordinate_name): - return self.coordinate_dict[coordinate_name] +# def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): +# # Following values don't matter for test. Just needed as arguments +# grid_type = um2nc.GRID_NEW_DYNAMICS +# dlat = 1.25 +# dlon = 1.875 -@pytest.fixture -def ua_plev_cube_with_latlon_coords(ua_plev_cube): - lat_points = np.array([-90., -88.75, -87.5], dtype="float32") - lon_points = np.array([0., 1.875, 3.75], dtype="float32") +# lat_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LAT_COORD_NAME) +# lon_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LON_COORD_NAME) - lat_coord_object = DummyCoordinate( - um2nc.LAT_COORD_NAME, - lat_points - ) - lon_coord_object = DummyCoordinate( - um2nc.LON_COORD_NAME, - lon_points - ) +# assert lat_coord.points.dtype == np.dtype("float32") +# assert lon_coord.points.dtype == np.dtype("float32") - coords_dict = { - um2nc.LAT_COORD_NAME: lat_coord_object, - um2nc.LON_COORD_NAME: lon_coord_object - } +# assert not lat_coord.has_bounds() +# assert not lon_coord.has_bounds() - cube_with_coords = DummyCubeWithCoords( - ua_plev_cube.item_code, - ua_plev_cube.var_name, - coords=coords_dict - ) +# # Mock additional functions called by um2nc.fix_latlon_coords, as they may +# # require methods not implemented by the DummyCubeWithCoordinates. +# with ( +# mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value=None), +# mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value=None), +# mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value=None) +# ): +# um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) - return cube_with_coords +# assert lat_coord.points.dtype == np.dtype("float64") +# assert lon_coord.points.dtype == np.dtype("float64") -def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): - # Test that coordinate arrays are converted to float64 - # Following values don't matter for test. Just needed as arguments - grid_type = um2nc.GRID_NEW_DYNAMICS - dlat = 1.25 - dlon = 1.875 - lat_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LAT_COORD_NAME) - lon_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LON_COORD_NAME) - assert lat_coord.points.dtype == np.dtype("float32") - assert lon_coord.points.dtype == np.dtype("float32") +# def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): +# # Test that coordinate arrays are converted to float64 - # Mock additional functions called by um2nc.fix_latlon_coords, as they may - # require methods not implemented by the DummyCubeWithCoordinates. - with ( - mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value=None), - mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value=None), - mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value=None) - ): - um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) +# # Following values don't matter for test. Just needed as arguments +# grid_type = um2nc.GRID_NEW_DYNAMICS +# dlat = 1.25 +# dlon = 1.875 - assert lat_coord.points.dtype == np.dtype("float64") - assert lon_coord.points.dtype == np.dtype("float64") +# lat_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LAT_COORD_NAME) +# lon_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LON_COORD_NAME) +# assert lat_coord.points.dtype == np.dtype("float32") +# assert lon_coord.points.dtype == np.dtype("float32") -def test_fix_latlon_coords_error(ua_plev_cube_with_latlon_coords): - # Test that fix_latlon_coords raises the right type of error when a cube - # is missing coordinates - def _raise_CoordinateNotFoundError(coord_name): - # Include an argument "coord_name" to mimic the signature of an - # Iris cube's ".coord" method - raise iris.exceptions.CoordinateNotFoundError(coord_name) +# # Mock additional functions called by um2nc.fix_latlon_coords, as they may +# # require methods not implemented by the DummyCubeWithCoordinates. +# with ( +# mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value=None), +# mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value=None), +# mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value=None) +# ): +# um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) - # Following values don't matter for test. Just needed as arguments - grid_type = um2nc.GRID_NEW_DYNAMICS - dlat = 1.25 - dlon = 1.875 +# assert lat_coord.points.dtype == np.dtype("float64") +# assert lon_coord.points.dtype == np.dtype("float64") - # Replace coord method to raise UnsupportedTimeSeriesError - ua_plev_cube_with_latlon_coords.coord = _raise_CoordinateNotFoundError - with ( - pytest.raises(um2nc.UnsupportedTimeSeriesError) - ): - um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) +# def test_fix_latlon_coords_timeseries_error(ua_plev_cube_with_latlon_coords): +# # Test that fix_latlon_coords raises the right type of error when a cube +# # is missing latitude or longitude coordinates. +# ua_plev_cube_with_latlon_coords.coords + + +# def _raise_CoordinateNotFoundError(coord_name): +# # Include an argument "coord_name" to mimic the signature of an +# # Iris cube's ".coord" method +# raise iris.exceptions.CoordinateNotFoundError(coord_name) + +# # Following values don't matter for test. Just needed as arguments +# grid_type = um2nc.GRID_NEW_DYNAMICS +# dlat = 1.25 +# dlon = 1.875 + +# # Replace coord method to raise UnsupportedTimeSeriesError +# ua_plev_cube_with_latlon_coords.coord = _raise_CoordinateNotFoundError + +# with ( +# pytest.raises(um2nc.UnsupportedTimeSeriesError) +# ): +# um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 61b5420..e234931 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -289,7 +289,6 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): dlon: (float) zonal spacing between longitude grid points. """ - # TODO: Check that we don't need the double coordinate lookup. try: latitude_coordinate = cube.coord(LAT_COORD_NAME) longitude_coordinate = cube.coord(LON_COORD_NAME) From 62a1bec829df6bf82cebe788bcb1a1d35a4ad8b8 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Sat, 17 Aug 2024 23:29:00 +1000 Subject: [PATCH 14/27] Simplify DummyCoordinate, add tests for specific grid types --- test/test_um2netcdf.py | 620 ++++++++++++++++++++++++++--------------- umpost/um2netcdf.py | 1 - 2 files changed, 396 insertions(+), 225 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 94d00ac..973b973 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -13,6 +13,9 @@ import mule.ff import iris.cube +D_LAT_N96 = 1.25 +D_LON_N96 = 1.875 + @pytest.fixture def z_sea_rho_data(): @@ -658,136 +661,6 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units -# @pytest.fixture -# def lat_v_points_dlat_EG(): -# # Array of latitude points and corresponding spacing imitating -# # the real lat_v grid from CM2 which uses grid type EG. - -# dlat = 1.25 -# lat_v_points = np.arange(-90., 91, dlat) -# return (lat_v_points, um2nc.GRID_END_GAME, dlat) - - -# @pytest.fixture -# def lon_u_points_dlon_EG(): -# # Array of longitude points and corresponding spacing imitating -# # the real lon_u grid from CM2 which uses grid type EG. - -# dlon = 1.875 -# lon_u_points = np.arange(0, 360, dlon) -# return (lon_u_points, um2nc.GRID_END_GAME, dlon) - - -# @pytest.fixture -# def lat_points_standard_dlat_ND(): -# # Array of latitude points and corresponding spacing imitating the -# # standard (not river or v) lat grid for ESM1.5 which uses -# # grid type ND. - -# dlat = 1.25 -# lat_points_middle = np.arange(-88.75, 89., 1.25) -# lat_points = np.concatenate(([-90], -# lat_points_middle, -# [90.] -# )) -# return (lat_points, um2nc.GRID_NEW_DYNAMICS, dlat) - - -# @pytest.fixture -# def lon_points_standard_dlon_ND(): -# # Array of longitude points and corresponding spacing imitating the -# # standard (not river or u) lon grid for ESM1.5 which uses grid -# # type ND. - -# dlon = 1.875 -# lon_points = np.arange(0, 360, dlon) -# return (lon_points, um2nc.GRID_NEW_DYNAMICS, dlon) - - -# @pytest.fixture -# def lat_points_standard_dlat_EG(): -# # Array of latitude points and corresponding spacing imitating the -# # standard (not river or v) lat grid for CM2 which uses -# # grid type EG. - -# dlat = 1.25 -# lat_points = np.arange(-90 + 0.5*dlat, 90., dlat) -# return (lat_points, um2nc.GRID_END_GAME, dlat) - - -# @pytest.fixture -# def lon_points_standard_dlon_EG(): -# # Array of longitude points and corresponding spacing imitating the -# # standard (not river or u) lon grid for CM2 which uses grid -# # type EG. - -# dlon = 1.875 -# lon_points = np.arange(0.5*dlon, 360, dlon) -# return (lon_points, um2nc.GRID_END_GAME, dlon) - - -# def test_is_lat_river_grid(lat_river_points, lat_points_standard_dlat_ND): -# assert len(lat_river_points) == um2nc.NUM_LAT_RIVER_GRID_POINTS -# assert um2nc.is_lat_river_grid(lat_river_points) - -# # latitude points on ESM1.5 N96 -# not_lat_river_points = lat_points_standard_dlat_ND[0] - -# assert len(not_lat_river_points) != um2nc.NUM_LAT_RIVER_GRID_POINTS -# assert not um2nc.is_lat_river_grid(not_lat_river_points) - - -# def test_is_lon_river_grid(lon_river_points, lon_points_standard_dlon_ND): -# assert len(lon_river_points) == um2nc.NUM_LON_RIVER_GRID_POINTS -# assert um2nc.is_lon_river_grid(lon_river_points) - -# # longitude points on normal ESM1.5 N96 grid -# not_lon_river_points = lon_points_standard_dlon_ND[0] -# assert len(not_lon_river_points) != um2nc.NUM_LON_RIVER_GRID_POINTS -# assert not um2nc.is_lon_river_grid(not_lon_river_points) - - -# def test_is_lat_v_grid(lat_v_points_dlat_EG, -# lat_v_points_dlat_ND, -# lat_points_standard_dlat_ND, -# lat_points_standard_dlat_EG -# ): -# lat_v_points, grid_code, dlat = lat_v_points_dlat_EG -# assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) - -# lat_v_points, grid_code, dlat = lat_v_points_dlat_ND -# assert um2nc.is_lat_v_grid(lat_v_points, grid_code, dlat) - -# not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_EG -# assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) - -# not_lat_v_points, grid_code, dlat = lat_points_standard_dlat_ND -# assert not um2nc.is_lat_v_grid(not_lat_v_points, grid_code, dlat) - - -# def test_is_lon_u_grid(lon_u_points_dlon_EG, -# lon_u_points_dlon_ND, -# lon_points_standard_dlon_ND, -# lon_points_standard_dlon_EG -# ): -# lon_u_points, grid_code, dlon = lon_u_points_dlon_EG -# assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) - -# lon_u_points, grid_code, dlon = lon_u_points_dlon_ND -# assert um2nc.is_lon_u_grid(lon_u_points, grid_code, dlon) - -# not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_EG -# assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) - -# not_lon_u_points, grid_code, dlon = lon_points_standard_dlon_ND -# assert not um2nc.is_lon_u_grid(not_lon_u_points, grid_code, dlon) - -# Completely faked bounds to be added to a DummyCoordinate when -# its guess_bounds method is called. - -DUMMY_GUESSED_BOUNDS = "dummy_guessed_bounds" - - @dataclass class DummyCoordinate: """ @@ -799,69 +672,39 @@ class DummyCoordinate: """ coordname: str points: np.ndarray - # NB: MuleVars is not part of an iris coordinate, but is packaged - # in here to aid with testing. - mulevars: um2nc.MuleVars - - bounds = None + bounds: np.ndarray = None # Note that var_name attribute is different thing to return # value of name() method. var_name: str = None - # Expected attributes after testing: - expected_var_name: str = None - expected_points_type: str = None - expected_bounds: any = None - def name(self): return self.coordname def has_bounds(self): return self.bounds is not None - def guess_bounds(self): - self.bounds = DUMMY_GUESSED_BOUNDS @pytest.fixture def lat_river_dummy(): # Dummy coordinate imitating UM V7.3s river grid. Must have length 180. lat_river_points = np.arange(-90., 90, dtype="float32") + 0.5 - d_lat = 1.25 - d_lon = 1.875 - mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) - lat_river_dummy_coord = DummyCoordinate( coordname=um2nc.LAT_COORD_NAME, - points=lat_river_points, - mulevars=mv, - # Expected values after fix_latlon_tests - expected_var_name="lat_river", - expected_points_type=np.dtype("float64"), - expected_bounds=DUMMY_GUESSED_BOUNDS + points=lat_river_points ) - return lat_river_dummy_coord + @pytest.fixture def lon_river_dummy(): # Dummy coordinate imitating UM V7.3s river grid. Must have length 90. lon_river_points = np.arange(0., 360., dtype="float32") + 0.5 - d_lat = 1.25 - d_lon = 1.875 - mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) - lon_river_dummy_coord = DummyCoordinate( coordname=um2nc.LON_COORD_NAME, points=lon_river_points, - mulevars=mv, - # Expected values after fix_latlon_tests - expected_var_name="lon_river", - expected_points_type=np.dtype("float64"), - expected_bounds=DUMMY_GUESSED_BOUNDS ) - return lon_river_dummy_coord @@ -869,48 +712,107 @@ def lon_river_dummy(): def lat_v_nd_dummy(): # Dummy coordinate imitating imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). - - d_lat = 1.25 - d_lon = 1.875 - lat_v_points = np.arange(-90.+0.5*d_lat, 90, d_lat, dtype="float32") - - mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) + lat_v_points = np.arange(-90.+0.5*D_LAT_N96, 90, D_LAT_N96, dtype="float32") lat_v_nd_dummy_coord = DummyCoordinate( coordname=um2nc.LAT_COORD_NAME, points=lat_v_points, - mulevars=mv, - # Expected values after fix_latlon_tests - expected_var_name="lat_v", - expected_points_type=np.dtype("float64"), - expected_bounds=DUMMY_GUESSED_BOUNDS ) - return lat_v_nd_dummy_coord @pytest.fixture def lon_u_nd_dummy(): - # Dummy coordinate imitating imitating the real + # Dummy coordinate imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). - - d_lat = 1.25 - d_lon = 1.875 - lon_u_points = np.arange(0.5*d_lon, 360, d_lon, dtype="float32") - - mv = um2nc.MuleVars(um2nc.GRID_NEW_DYNAMICS, d_lat, d_lon, [], []) + lon_u_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") lon_u_nd_dummy_coord = DummyCoordinate( coordname=um2nc.LON_COORD_NAME, points=lon_u_points, - mulevars=mv, - # Expected values after fix_latlon_tests - expected_var_name="lon_u", - expected_points_type=np.dtype("float64"), - expected_bounds=DUMMY_GUESSED_BOUNDS + ) + return lon_u_nd_dummy_coord + +@pytest.fixture +def lat_v_eg_dummy(): + # Dummy coordinate imitating the real + # lat_v grid from CM2 (which uses the End Game grid). + lat_v_points = np.arange(-90., 91, D_LAT_N96, dtype="float32") + + lat_v_eg_dummy_coord = DummyCoordinate( + coordname=um2nc.LAT_COORD_NAME, + points=lat_v_points ) + return lat_v_eg_dummy_coord - return lon_u_nd_dummy_coord + +@pytest.fixture +def lon_u_eg_dummy(): + # Dummy coordinate imitating the real + # lon_v grid from CM2 (which uses the End Game grid). + lon_u_points = np.arange(0, 360, D_LON_N96, dtype="float32") + + lon_u_eg_dummy_coord = DummyCoordinate( + coordname=um2nc.LON_COORD_NAME, + points=lon_u_points + ) + return lon_u_eg_dummy_coord + + +@pytest.fixture +def lat_standard_nd_dummy(): + # Dummy coordinate imitating the standard latitude + # grid from ESM1.5 (which uses the New Dynamics grid). + lat_points_middle = np.arange(-88.75, 89., D_LAT_N96) + lat_points = np.concatenate(([-90], + lat_points_middle, + [90.] + ), dtype="float32") + + lat_standard_nd_dummy_coord = DummyCoordinate( + coordname=um2nc.LAT_COORD_NAME, + points=lat_points + ) + return lat_standard_nd_dummy_coord + + +@pytest.fixture +def lon_standard_nd_dummy(): + # Dummy coordinate imitating the standard longitude + # grid from ESM1.5 (which uses the New Dynamics grid). + lon_points = np.arange(0, 360, D_LON_N96, dtype="float32") + + lon_standard_nd_dummy_coord = DummyCoordinate( + coordname=um2nc.LON_COORD_NAME, + points=lon_points + ) + return lon_standard_nd_dummy_coord + + +@pytest.fixture +def lat_standard_eg_dummy(): + # Dummy coordinate imitating the standard latitude + # grid from CM2 (which uses the End Game grid). + lat_points = np.arange(-90 + 0.5*D_LAT_N96, 90., + D_LAT_N96, dtype="float32") + + lat_standard_eg_dummy_coord = DummyCoordinate( + coordname=um2nc.LAT_COORD_NAME, + points=lat_points + ) + return lat_standard_eg_dummy_coord + + +@pytest.fixture +def lon_standard_eg_dummy(): + # Dummy coordinate imitating the standard longitude + # grid from CM2 (which uses the End Game grid). + lon_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") + lon_standard_eg_dummy_coord = DummyCoordinate( + coordname=um2nc.LON_COORD_NAME, + points=lon_points + ) + return lon_standard_eg_dummy_coord class DummyCubeWithCoords(DummyCube): @@ -924,62 +826,332 @@ def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] -def test_fix_latlon_coords(ua_plev_cube, - lat_river_dummy, - lon_river_dummy, - lat_v_nd_dummy, - lon_u_nd_dummy): +# Tests of fix_latlon_coords. This function converts coordinate points +# to double, adds bounds, and adds var_names to the coordinates. +# The following tests check that these are done correctly. +def test_fix_latlon_coords_river(ua_plev_cube, + lat_river_dummy, + lon_river_dummy): + """ + Tests of the fix_lat_lon_coords function on river grid coordinates. """ - Tests of the fix_lat_lon_coords function. - fix_latlon_coords makes the following modifications in place: - - Converts coordinate points to double. - - Adds bounds to its coordinates. - - Adds var_name attributes to the coordinates. - Here we test that these three things are done correctly. + cube_with_river_coords = DummyCubeWithCoords( + item_code=ua_plev_cube.item_code, + var_name=ua_plev_cube.var_name, + coords={ + um2nc.LAT_COORD_NAME: lat_river_dummy, + um2nc.LON_COORD_NAME: lon_river_dummy + } + ) + + # Supplementary grid information used by fix_latlon_coords. + # Grid type, d_lat, d_lon don't matter for river grid, but must be set. + d_lat = D_LAT_N96 + d_lon = D_LON_N96 + grid_type = um2nc.GRID_END_GAME + + cube_lat_coord = cube_with_river_coords.coord(um2nc.LAT_COORD_NAME) + cube_lon_coord = cube_with_river_coords.coord(um2nc.LON_COORD_NAME) + + # Checks prior to modifications. + assert cube_lat_coord.points.dtype == np.dtype("float32") + assert cube_lon_coord.points.dtype == np.dtype("float32") + + assert not cube_lat_coord.has_bounds() + assert not cube_lon_coord.has_bounds() + + assert cube_lat_coord.var_name is None + assert cube_lon_coord.var_name is None + + # Add mocks of guess_bounds method, to see that it is called. + cube_lat_coord.guess_bounds = mock.Mock(return_value=None) + cube_lon_coord.guess_bounds = mock.Mock(return_value=None) + + um2nc.fix_latlon_coords(cube_with_river_coords, grid_type, d_lat, d_lon) + + expected_lon_name = "lon_river" + expected_lat_name = "lat_river" + expected_dtype = np.dtype("float64") + + # Checks post modifications. + assert cube_lat_coord.var_name == expected_lat_name + assert cube_lon_coord.var_name == expected_lon_name + + assert cube_lat_coord.points.dtype == expected_dtype + assert cube_lon_coord.points.dtype == expected_dtype + + cube_lat_coord.guess_bounds.assert_called + cube_lon_coord.guess_bounds.assert_called + + +def test_fix_latlon_coords_uv(ua_plev_cube, + lat_v_nd_dummy, + lon_u_nd_dummy, + lat_v_eg_dummy, + lon_u_eg_dummy): """ - dummy_coord_pairs = [ - (lat_river_dummy, lon_river_dummy), - (lat_v_nd_dummy, lon_u_nd_dummy) + Tests of the fix_lat_lon_coords for longitude u and latitude v + coordinates on both the New Dynamics and End Game grids. + """ + # Both coordinate sets are on an N96 grid. + d_lat = D_LAT_N96 + d_lon = D_LON_N96 + + # Expected values after modification + expected_lat_name = "lat_v" + expected_lon_name = "lon_u" + expected_dtype = np.dtype("float64") + + coord_sets = [ + (lat_v_nd_dummy, lon_u_nd_dummy, um2nc.GRID_NEW_DYNAMICS), + (lat_v_eg_dummy, lon_u_eg_dummy, um2nc.GRID_END_GAME) ] - for lat_coord, lon_coord in dummy_coord_pairs: - # Check that we're providing a consistent coordinate pair. - assert lat_coord.mulevars == lon_coord.mulevars + for lat_coordinate, lon_coordinate, grid_type in coord_sets: + + cube_with_uv_coords = DummyCubeWithCoords( + item_code=ua_plev_cube.item_code, + var_name=ua_plev_cube.var_name, + coords={ + um2nc.LAT_COORD_NAME: lat_coordinate, + um2nc.LON_COORD_NAME: lon_coordinate + } + ) + + # Checks prior to modifications. + assert lat_coordinate.points.dtype == np.dtype("float32") + assert lon_coordinate.points.dtype == np.dtype("float32") + + assert not lat_coordinate.has_bounds() + assert not lon_coordinate.has_bounds() + + assert lat_coordinate.var_name is None + assert lon_coordinate.var_name is None + + # Add mocks of guess_bounds method, to see that it is called. + lat_coordinate.guess_bounds = mock.Mock(return_value=None) + lon_coordinate.guess_bounds = mock.Mock(return_value=None) - mv = lat_coord.mulevars + um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, + d_lat, d_lon) - # Construct a dummy cube object with the specified dummy coordinates. - cube_with_coords = DummyCubeWithCoords( + assert lat_coordinate.var_name == expected_lat_name + assert lon_coordinate.var_name == expected_lon_name + + assert lat_coordinate.points.dtype == expected_dtype + assert lon_coordinate.points.dtype == expected_dtype + + lat_coordinate.guess_bounds.assert_called + lon_coordinate.guess_bounds.assert_called + + +def test_fix_latlon_coords_standard(ua_plev_cube, + lat_standard_nd_dummy, + lon_standard_nd_dummy, + lat_standard_eg_dummy, + lon_standard_eg_dummy): + """ + Tests of the fix_lat_lon_coords for standard longitude + and latitude coordinates on both the New Dynamics and + End Game grids. + """ + # Both coordinate sets are on an N96 grid. + d_lat = D_LAT_N96 + d_lon = D_LON_N96 + + # Expected values after modification + expected_lat_name = "lat" + expected_lon_name = "lon" + expected_dtype = np.dtype("float64") + + coord_sets = [ + ( + lat_standard_nd_dummy, + lon_standard_nd_dummy, + um2nc.GRID_NEW_DYNAMICS + ), + ( + lat_standard_eg_dummy, + lon_standard_eg_dummy, + um2nc.GRID_END_GAME + ) + ] + + for lat_coordinate, lon_coordinate, grid_type in coord_sets: + + cube_with_uv_coords = DummyCubeWithCoords( item_code=ua_plev_cube.item_code, var_name=ua_plev_cube.var_name, coords={ - um2nc.LAT_COORD_NAME: lat_coord, - um2nc.LON_COORD_NAME: lon_coord + um2nc.LAT_COORD_NAME: lat_coordinate, + um2nc.LON_COORD_NAME: lon_coordinate } ) # Checks prior to modifications. - assert lat_coord.points.dtype == np.dtype("float32") - assert lon_coord.points.dtype == np.dtype("float32") + assert lat_coordinate.points.dtype == np.dtype("float32") + assert lon_coordinate.points.dtype == np.dtype("float32") + + assert not lat_coordinate.has_bounds() + assert not lon_coordinate.has_bounds() + + assert lat_coordinate.var_name is None + assert lon_coordinate.var_name is None + + # Add mocks of guess_bounds method, to see that it is called. + lat_coordinate.guess_bounds = mock.Mock(return_value=None) + lon_coordinate.guess_bounds = mock.Mock(return_value=None) + + um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, + d_lat, d_lon) + + assert lat_coordinate.var_name == expected_lat_name + assert lon_coordinate.var_name == expected_lon_name + + assert lat_coordinate.points.dtype == expected_dtype + assert lon_coordinate.points.dtype == expected_dtype + + lat_coordinate.guess_bounds.assert_called + lon_coordinate.guess_bounds.assert_called + + +def test_fix_latlon_coords_single_point(ua_plev_cube): + """ + Test that single point longitude and latitude coordinates + are provided with global bounds. + """ + d_lat = D_LAT_N96 + d_lon = D_LON_N96 + grid_type = um2nc.GRID_NEW_DYNAMICS + + # Expected values after modification + expected_lat_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LAT_COORD_NAME] + expected_lon_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LON_COORD_NAME] + + lat_coord_single = DummyCoordinate(coordname=um2nc.LAT_COORD_NAME, + points=np.array([0])) + lon_coord_single = DummyCoordinate(coordname=um2nc.LON_COORD_NAME, + points=np.array([0])) + + cube_with_uv_coords = DummyCubeWithCoords( + item_code=ua_plev_cube.item_code, + var_name=ua_plev_cube.var_name, + coords={ + um2nc.LAT_COORD_NAME: lat_coord_single, + um2nc.LON_COORD_NAME: lon_coord_single + } + ) + + assert not lat_coord_single.has_bounds() + assert not lon_coord_single.has_bounds() + + um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, + d_lat, d_lon) + + assert lat_coord_single.has_bounds() + assert lon_coord_single.has_bounds() + assert np.array_equal(lat_coord_single.bounds, expected_lat_bounds) + assert np.array_equal(lon_coord_single.bounds, expected_lon_bounds) + +def test_fix_latlon_coords_has_bounds(ua_plev_cube): + """ + Test that existing coordinate bounds are not modified by + fix_latlon_coords. + """ + # Grid information required to run fix_latlon_coords + d_lat = D_LAT_N96 + d_lon = D_LON_N96 + grid_type = um2nc.GRID_NEW_DYNAMICS + + # Expected values after modification + lon_bounds = np.array([[0, 1]]) + lat_bounds = np.array([[10, 25]]) + + lat_coord = DummyCoordinate(coordname=um2nc.LAT_COORD_NAME, + points=np.array([0]), + bounds=lat_bounds.copy()) + lon_coord = DummyCoordinate(coordname=um2nc.LON_COORD_NAME, + points=np.array([0]), + bounds=lon_bounds.copy()) + + cube_with_uv_coords = DummyCubeWithCoords( + item_code=ua_plev_cube.item_code, + var_name=ua_plev_cube.var_name, + coords={ + um2nc.LAT_COORD_NAME: lat_coord, + um2nc.LON_COORD_NAME: lon_coord + } + ) + assert lat_coord.has_bounds() + assert lon_coord.has_bounds() + + um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, + d_lat, d_lon) + + assert np.array_equal(lat_coord.bounds, lat_bounds) + assert np.array_equal(lon_coord.bounds, lon_bounds) + + + + + +# def test_fix_latlon_coords(ua_plev_cube, +# lat_river_dummy, +# lon_river_dummy, +# lat_v_nd_dummy, +# lon_u_nd_dummy): +# """ +# Tests of the fix_lat_lon_coords function. +# fix_latlon_coords makes the following modifications in place: +# - Converts coordinate points to double. +# - Adds bounds to its coordinates. +# - Adds var_name attributes to the coordinates. + +# Here we test that these three things are done correctly. +# """ +# dummy_coord_pairs = [ +# (lat_river_dummy, lon_river_dummy), +# (lat_v_nd_dummy, lon_u_nd_dummy) +# ] +# for lat_coord, lon_coord in dummy_coord_pairs: + +# # Check that we're providing a consistent coordinate pair. +# assert lat_coord.mulevars == lon_coord.mulevars + +# mv = lat_coord.mulevars + +# # Construct a dummy cube object with the specified dummy coordinates. +# cube_with_coords = DummyCubeWithCoords( +# item_code=ua_plev_cube.item_code, +# var_name=ua_plev_cube.var_name, +# coords={ +# um2nc.LAT_COORD_NAME: lat_coord, +# um2nc.LON_COORD_NAME: lon_coord +# } +# ) + +# # Checks prior to modifications. +# assert lat_coord.points.dtype == np.dtype("float32") +# assert lon_coord.points.dtype == np.dtype("float32") - assert not lat_coord.has_bounds() - assert not lon_coord.has_bounds() +# assert not lat_coord.has_bounds() +# assert not lon_coord.has_bounds() - assert lat_coord.var_name is None - assert lon_coord.var_name is None +# assert lat_coord.var_name is None +# assert lon_coord.var_name is None - um2nc.fix_latlon_coords(cube_with_coords, mv.grid_type, mv.d_lat, mv.d_lon) +# um2nc.fix_latlon_coords(cube_with_coords, mv.grid_type, mv.d_lat, mv.d_lon) - # Checks post modifications. - assert lat_coord.var_name == lat_coord.expected_var_name - assert lon_coord.var_name == lon_coord.expected_var_name +# # Checks post modifications. +# assert lat_coord.var_name == lat_coord.expected_var_name +# assert lon_coord.var_name == lon_coord.expected_var_name - assert lat_coord.points.dtype == lat_coord.expected_points_type - assert lon_coord.points.dtype == lon_coord.expected_points_type +# assert lat_coord.points.dtype == lat_coord.expected_points_type +# assert lon_coord.points.dtype == lon_coord.expected_points_type - assert lat_coord.bounds == lat_coord.expected_bounds - assert lon_coord.bounds == lon_coord.expected_bounds +# assert lat_coord.bounds == lat_coord.expected_bounds +# assert lon_coord.bounds == lon_coord.expected_bounds # def test_add_latlon_coord_bounds_has_bounds(): diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index e234931..f356488 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -211,7 +211,6 @@ def is_lat_v_grid(latitude_points, grid_type, dlat): grid_type: (string) model horizontal grid type. dlat: (float) meridional spacing between latitude grid points. """ - min_latitude = latitude_points[0] min_lat_v_nd_grid = -90.+0.5*dlat min_lat_v_eg_grid = -90 From 317beaed64acea8ad346423cbe72680d90465e8e Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Sat, 17 Aug 2024 23:54:27 +1000 Subject: [PATCH 15/27] Simplify DummyCubeWithCoords, ensure that coordinate names match between DummyCubeWithCoords and DummyCoordinate --- test/test_um2netcdf.py | 406 +++-------------------------------------- 1 file changed, 28 insertions(+), 378 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 973b973..12070c2 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -712,7 +712,8 @@ def lon_river_dummy(): def lat_v_nd_dummy(): # Dummy coordinate imitating imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). - lat_v_points = np.arange(-90.+0.5*D_LAT_N96, 90, D_LAT_N96, dtype="float32") + lat_v_points = np.arange(-90.+0.5*D_LAT_N96, 90, + D_LAT_N96, dtype="float32") lat_v_nd_dummy_coord = DummyCoordinate( coordname=um2nc.LAT_COORD_NAME, @@ -733,6 +734,7 @@ def lon_u_nd_dummy(): ) return lon_u_nd_dummy_coord + @pytest.fixture def lat_v_eg_dummy(): # Dummy coordinate imitating the real @@ -818,9 +820,19 @@ def lon_standard_eg_dummy(): class DummyCubeWithCoords(DummyCube): # DummyCube with coordinates, which can be filled with # DummyCoordinate objects for testing. - def __init__(self, item_code, var_name=None, attributes=None, units=None, coords={}): - super().__init__(item_code, var_name, attributes, units) - self.coordinate_dict = coords + def __init__(self, dummy_cube, coords=[]): + super().__init__(dummy_cube.item_code, + dummy_cube.var_name, + dummy_cube.attributes, + dummy_cube.units) + + # Set up coordinate dictionary to hold each dummy coordinate, with keys + # given by the coordinate names. This ensures that the key used to + # access a coordinate via the coord method matches the + # coordinate's name. + self.coordinate_dict = {} + for dummy_coord in coords: + self.coordinate_dict[dummy_coord.name()] = dummy_coord def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] @@ -837,12 +849,8 @@ def test_fix_latlon_coords_river(ua_plev_cube, """ cube_with_river_coords = DummyCubeWithCoords( - item_code=ua_plev_cube.item_code, - var_name=ua_plev_cube.var_name, - coords={ - um2nc.LAT_COORD_NAME: lat_river_dummy, - um2nc.LON_COORD_NAME: lon_river_dummy - } + dummy_cube=ua_plev_cube, + coords=[lat_river_dummy, lon_river_dummy] ) # Supplementary grid information used by fix_latlon_coords. @@ -911,12 +919,8 @@ def test_fix_latlon_coords_uv(ua_plev_cube, for lat_coordinate, lon_coordinate, grid_type in coord_sets: cube_with_uv_coords = DummyCubeWithCoords( - item_code=ua_plev_cube.item_code, - var_name=ua_plev_cube.var_name, - coords={ - um2nc.LAT_COORD_NAME: lat_coordinate, - um2nc.LON_COORD_NAME: lon_coordinate - } + dummy_cube=ua_plev_cube, + coords=[lat_coordinate, lon_coordinate] ) # Checks prior to modifications. @@ -981,12 +985,8 @@ def test_fix_latlon_coords_standard(ua_plev_cube, for lat_coordinate, lon_coordinate, grid_type in coord_sets: cube_with_uv_coords = DummyCubeWithCoords( - item_code=ua_plev_cube.item_code, - var_name=ua_plev_cube.var_name, - coords={ - um2nc.LAT_COORD_NAME: lat_coordinate, - um2nc.LON_COORD_NAME: lon_coordinate - } + dummy_cube=ua_plev_cube, + coords=[lat_coordinate, lon_coordinate] ) # Checks prior to modifications. @@ -1035,12 +1035,8 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): points=np.array([0])) cube_with_uv_coords = DummyCubeWithCoords( - item_code=ua_plev_cube.item_code, - var_name=ua_plev_cube.var_name, - coords={ - um2nc.LAT_COORD_NAME: lat_coord_single, - um2nc.LON_COORD_NAME: lon_coord_single - } + dummy_cube=ua_plev_cube, + coords=[lat_coord_single, lon_coord_single] ) assert not lat_coord_single.has_bounds() @@ -1054,9 +1050,10 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): assert np.array_equal(lat_coord_single.bounds, expected_lat_bounds) assert np.array_equal(lon_coord_single.bounds, expected_lon_bounds) + def test_fix_latlon_coords_has_bounds(ua_plev_cube): """ - Test that existing coordinate bounds are not modified by + Test that existing coordinate bounds are not modified by fix_latlon_coords. """ # Grid information required to run fix_latlon_coords @@ -1076,12 +1073,8 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): bounds=lon_bounds.copy()) cube_with_uv_coords = DummyCubeWithCoords( - item_code=ua_plev_cube.item_code, - var_name=ua_plev_cube.var_name, - coords={ - um2nc.LAT_COORD_NAME: lat_coord, - um2nc.LON_COORD_NAME: lon_coord - } + dummy_cube=ua_plev_cube, + coords=[lat_coord, lon_coord] ) assert lat_coord.has_bounds() assert lon_coord.has_bounds() @@ -1091,346 +1084,3 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): assert np.array_equal(lat_coord.bounds, lat_bounds) assert np.array_equal(lon_coord.bounds, lon_bounds) - - - - - -# def test_fix_latlon_coords(ua_plev_cube, -# lat_river_dummy, -# lon_river_dummy, -# lat_v_nd_dummy, -# lon_u_nd_dummy): -# """ -# Tests of the fix_lat_lon_coords function. -# fix_latlon_coords makes the following modifications in place: -# - Converts coordinate points to double. -# - Adds bounds to its coordinates. -# - Adds var_name attributes to the coordinates. - -# Here we test that these three things are done correctly. -# """ -# dummy_coord_pairs = [ -# (lat_river_dummy, lon_river_dummy), -# (lat_v_nd_dummy, lon_u_nd_dummy) -# ] -# for lat_coord, lon_coord in dummy_coord_pairs: - -# # Check that we're providing a consistent coordinate pair. -# assert lat_coord.mulevars == lon_coord.mulevars - -# mv = lat_coord.mulevars - -# # Construct a dummy cube object with the specified dummy coordinates. -# cube_with_coords = DummyCubeWithCoords( -# item_code=ua_plev_cube.item_code, -# var_name=ua_plev_cube.var_name, -# coords={ -# um2nc.LAT_COORD_NAME: lat_coord, -# um2nc.LON_COORD_NAME: lon_coord -# } -# ) - -# # Checks prior to modifications. -# assert lat_coord.points.dtype == np.dtype("float32") -# assert lon_coord.points.dtype == np.dtype("float32") - -# assert not lat_coord.has_bounds() -# assert not lon_coord.has_bounds() - -# assert lat_coord.var_name is None -# assert lon_coord.var_name is None - -# um2nc.fix_latlon_coords(cube_with_coords, mv.grid_type, mv.d_lat, mv.d_lon) - -# # Checks post modifications. -# assert lat_coord.var_name == lat_coord.expected_var_name -# assert lon_coord.var_name == lon_coord.expected_var_name - -# assert lat_coord.points.dtype == lat_coord.expected_points_type -# assert lon_coord.points.dtype == lon_coord.expected_points_type - -# assert lat_coord.bounds == lat_coord.expected_bounds -# assert lon_coord.bounds == lon_coord.expected_bounds - - -# def test_add_latlon_coord_bounds_has_bounds(): -# # Test that bounds are not modified if they already exist -# lon_points = np.array([1., 2., 3.]) -# lon_bounds = np.array([[0.5, 1.5], -# [1.5, 2.5], -# [2.5, 3.5]]) -# lon_coord_with_bounds = DummyCoordinate( -# um2nc.LON_COORD_NAME, -# lon_points, -# lon_bounds -# ) -# assert lon_coord_with_bounds.has_bounds() - -# um2nc.add_latlon_coord_bounds(lon_coord_with_bounds) -# assert np.array_equal(lon_coord_with_bounds.bounds, lon_bounds) - - -# def test_add_latlon_coord_guess_bounds(): -# # Test that guess_bounds method is called when -# # coordinate has no bounds and length > 1. -# lon_points = np.array([0., 1.]) -# lon_coord_nobounds = DummyCoordinate( -# um2nc.LON_COORD_NAME, -# lon_points -# ) - -# # Mock Iris' guess_bounds method to check whether it is called -# lon_coord_nobounds.guess_bounds = mock.Mock(return_value=None) - -# assert len(lon_coord_nobounds.points) > 1 -# assert not lon_coord_nobounds.has_bounds() - -# um2nc.add_latlon_coord_bounds(lon_coord_nobounds) - -# lon_coord_nobounds.guess_bounds.assert_called() - - -# def test_add_latlon_coord_single(): -# # Test that the correct global bounds are added to coordinates -# # with just a single point. -# for coord_name in [um2nc.LON_COORD_NAME, um2nc.LAT_COORD_NAME]: -# points = np.array([0.]) -# coord_single_point = DummyCoordinate( -# coord_name, -# points -# ) - -# assert len(coord_single_point.points) == 1 -# assert not coord_single_point.has_bounds() - -# um2nc.add_latlon_coord_bounds(coord_single_point) - -# expected_bounds = um2nc.GLOBAL_COORD_BOUNDS[coord_name] -# assert np.array_equal(coord_single_point.bounds, expected_bounds) - - -# def test_add_latlon_coord_error(): -# fake_coord_name = "fake coordinate" -# fake_points = np.array([1., 2., 3.]) - -# fake_coord = DummyCoordinate( -# fake_coord_name, -# fake_points -# ) - -# with pytest.raises(ValueError): -# um2nc.add_latlon_coord_bounds(fake_coord) - - -# def test_fix_lat_coord_name(): -# # Following values are ignored due to mocking of checking functions. -# grid_type = um2nc.GRID_END_GAME -# dlat = 1.875 -# lat_points = np.array([1., 2., 3.]) - -# latitude_coordinate = DummyCoordinate( -# um2nc.LAT_COORD_NAME, -# lat_points -# ) -# assert latitude_coordinate.var_name is None - -# # Mock the return value of grid checking functions in order to simplify test setup, -# # since grid checking functions have their own tests. -# with mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=True): -# um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) -# assert latitude_coordinate.var_name == "lat_river" - -# latitude_coordinate.var_name = None -# with ( -# mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), -# mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=True) -# ): -# um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) -# assert latitude_coordinate.var_name == "lat_v" - -# latitude_coordinate.var_name = None -# with ( -# mock.patch("umpost.um2netcdf.is_lat_river_grid", return_value=False), -# mock.patch("umpost.um2netcdf.is_lat_v_grid", return_value=False) -# ): -# um2nc.fix_lat_coord_name(latitude_coordinate, grid_type, dlat) -# assert latitude_coordinate.var_name == "lat" - - -# def test_fix_lon_coord_name(): -# # Following values are ignored due to mocking of checking functions. -# grid_type = um2nc.GRID_END_GAME -# dlon = 1.875 -# lon_points = np.array([1., 2., 3.]) - -# longitude_coordinate = DummyCoordinate( -# um2nc.LON_COORD_NAME, -# lon_points -# ) -# assert longitude_coordinate.var_name is None - -# # Mock the return value of grid checking functions in order to simplify test setup, -# # since grid checking functions have their own tests. -# with mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=True): -# um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) -# assert longitude_coordinate.var_name == "lon_river" - -# longitude_coordinate.var_name = None -# with ( -# mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), -# mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=True) -# ): -# um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) -# assert longitude_coordinate.var_name == "lon_u" - -# longitude_coordinate.var_name = None -# with ( -# mock.patch("umpost.um2netcdf.is_lon_river_grid", return_value=False), -# mock.patch("umpost.um2netcdf.is_lon_u_grid", return_value=False) -# ): -# um2nc.fix_lon_coord_name(longitude_coordinate, grid_type, dlon) -# assert longitude_coordinate.var_name == "lon" - - -# @pytest.fixture -# def coordinate_fake_name(): -# # Fake dummy coordinate with made up name to test that exceptions are raised -# fake_coord_name = "fake coordinate" -# fake_points = np.array([1., 2., 3.]) - -# fake_coord = DummyCoordinate( -# fake_coord_name, -# fake_points -# ) - -# return fake_coord - - -# def test_fix_lat_coord_name_error(coordinate_fake_name): -# # Following values unimportant. Just needed as function arguments. -# grid_type = um2nc.GRID_NEW_DYNAMICS -# dlat = 1.25 -# with pytest.raises(ValueError): -# um2nc.fix_lat_coord_name(coordinate_fake_name, grid_type, dlat) - - -# def test_fix_lon_coord_name_error(coordinate_fake_name): -# # Following values unimportant. Just needed as function arguments. -# grid_type = um2nc.GRID_NEW_DYNAMICS -# dlon = 1.875 -# with pytest.raises(ValueError): -# um2nc.fix_lon_coord_name(coordinate_fake_name, grid_type, dlon) - - - -# @pytest.fixture -# def ua_plev_cube_with_latlon_coords(ua_plev_cube): -# lat_points = np.array([-90., -88.75, -87.5], dtype="float32") -# lon_points = np.array([0., 1.875, 3.75], dtype="float32") - -# lat_coord_object = DummyCoordinate( -# um2nc.LAT_COORD_NAME, -# lat_points -# ) -# lon_coord_object = DummyCoordinate( -# um2nc.LON_COORD_NAME, -# lon_points -# ) - -# coords_dict = { -# um2nc.LAT_COORD_NAME: lat_coord_object, -# um2nc.LON_COORD_NAME: lon_coord_object -# } - -# cube_with_coords = DummyCubeWithCoords( -# ua_plev_cube.item_code, -# ua_plev_cube.var_name, -# coords=coords_dict -# ) - -# return cube_with_coords - - -# def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): - -# # Following values don't matter for test. Just needed as arguments -# grid_type = um2nc.GRID_NEW_DYNAMICS -# dlat = 1.25 -# dlon = 1.875 - -# lat_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LAT_COORD_NAME) -# lon_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LON_COORD_NAME) - -# assert lat_coord.points.dtype == np.dtype("float32") -# assert lon_coord.points.dtype == np.dtype("float32") - -# assert not lat_coord.has_bounds() -# assert not lon_coord.has_bounds() - -# # Mock additional functions called by um2nc.fix_latlon_coords, as they may -# # require methods not implemented by the DummyCubeWithCoordinates. -# with ( -# mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value=None), -# mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value=None), -# mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value=None) -# ): -# um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) - -# assert lat_coord.points.dtype == np.dtype("float64") -# assert lon_coord.points.dtype == np.dtype("float64") - - - - - -# def test_fix_latlon_coords_type_change(ua_plev_cube_with_latlon_coords): -# # Test that coordinate arrays are converted to float64 - -# # Following values don't matter for test. Just needed as arguments -# grid_type = um2nc.GRID_NEW_DYNAMICS -# dlat = 1.25 -# dlon = 1.875 - -# lat_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LAT_COORD_NAME) -# lon_coord = ua_plev_cube_with_latlon_coords.coord(um2nc.LON_COORD_NAME) - -# assert lat_coord.points.dtype == np.dtype("float32") -# assert lon_coord.points.dtype == np.dtype("float32") - -# # Mock additional functions called by um2nc.fix_latlon_coords, as they may -# # require methods not implemented by the DummyCubeWithCoordinates. -# with ( -# mock.patch("umpost.um2netcdf.add_latlon_coord_bounds", return_value=None), -# mock.patch("umpost.um2netcdf.fix_lat_coord_name", return_value=None), -# mock.patch("umpost.um2netcdf.fix_lon_coord_name", return_value=None) -# ): -# um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) - -# assert lat_coord.points.dtype == np.dtype("float64") -# assert lon_coord.points.dtype == np.dtype("float64") - - -# def test_fix_latlon_coords_timeseries_error(ua_plev_cube_with_latlon_coords): -# # Test that fix_latlon_coords raises the right type of error when a cube -# # is missing latitude or longitude coordinates. -# ua_plev_cube_with_latlon_coords.coords - - -# def _raise_CoordinateNotFoundError(coord_name): -# # Include an argument "coord_name" to mimic the signature of an -# # Iris cube's ".coord" method -# raise iris.exceptions.CoordinateNotFoundError(coord_name) - -# # Following values don't matter for test. Just needed as arguments -# grid_type = um2nc.GRID_NEW_DYNAMICS -# dlat = 1.25 -# dlon = 1.875 - -# # Replace coord method to raise UnsupportedTimeSeriesError -# ua_plev_cube_with_latlon_coords.coord = _raise_CoordinateNotFoundError - -# with ( -# pytest.raises(um2nc.UnsupportedTimeSeriesError) -# ): -# um2nc.fix_latlon_coords(ua_plev_cube_with_latlon_coords, grid_type, dlat, dlon) From 159e430c147dfe349aa5f4092d320862808729f1 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Mon, 19 Aug 2024 10:10:34 +1000 Subject: [PATCH 16/27] Add missing coordinate test --- test/test_um2netcdf.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 12070c2..461f4ec 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -1084,3 +1084,27 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): assert np.array_equal(lat_coord.bounds, lat_bounds) assert np.array_equal(lon_coord.bounds, lon_bounds) + + +def test_fix_latlon_coords_missing_coord_error(ua_plev_cube): + """ + Test that fix_latlon_coords raises the right type of error when a cube + is missing coordinates. + """ + def _raise_CoordinateNotFoundError(coord_name): + # Include an argument "coord_name" to mimic the signature of an + # Iris cube's ".coord" method + raise iris.exceptions.CoordinateNotFoundError(coord_name) + + # Following values don't matter for test. Just needed as arguments + grid_type = um2nc.GRID_NEW_DYNAMICS + dlat = 1.25 + dlon = 1.875 + + # Replace coord method to raise UnsupportedTimeSeriesError + ua_plev_cube.coord = _raise_CoordinateNotFoundError + + with ( + pytest.raises(um2nc.UnsupportedTimeSeriesError) + ): + um2nc.fix_latlon_coords(ua_plev_cube, grid_type, dlat, dlon) From ab3d6f7c511ccd4e23b36189f7b660bdd0ea9603 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Tue, 20 Aug 2024 09:37:23 +1000 Subject: [PATCH 17/27] Tidy up tests of fix_latlon_coords --- test/test_um2netcdf.py | 152 ++++++++++++++++------------------------- umpost/um2netcdf.py | 42 +++++++----- 2 files changed, 82 insertions(+), 112 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 461f4ec..1d27716 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -690,7 +690,7 @@ def lat_river_dummy(): lat_river_points = np.arange(-90., 90, dtype="float32") + 0.5 lat_river_dummy_coord = DummyCoordinate( - coordname=um2nc.LAT_COORD_NAME, + coordname=um2nc.LATITUDE, points=lat_river_points ) return lat_river_dummy_coord @@ -702,7 +702,7 @@ def lon_river_dummy(): lon_river_points = np.arange(0., 360., dtype="float32") + 0.5 lon_river_dummy_coord = DummyCoordinate( - coordname=um2nc.LON_COORD_NAME, + coordname=um2nc.LONGITUDE, points=lon_river_points, ) return lon_river_dummy_coord @@ -716,7 +716,7 @@ def lat_v_nd_dummy(): D_LAT_N96, dtype="float32") lat_v_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LAT_COORD_NAME, + coordname=um2nc.LATITUDE, points=lat_v_points, ) return lat_v_nd_dummy_coord @@ -729,7 +729,7 @@ def lon_u_nd_dummy(): lon_u_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") lon_u_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LON_COORD_NAME, + coordname=um2nc.LONGITUDE, points=lon_u_points, ) return lon_u_nd_dummy_coord @@ -742,7 +742,7 @@ def lat_v_eg_dummy(): lat_v_points = np.arange(-90., 91, D_LAT_N96, dtype="float32") lat_v_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LAT_COORD_NAME, + coordname=um2nc.LATITUDE, points=lat_v_points ) return lat_v_eg_dummy_coord @@ -755,7 +755,7 @@ def lon_u_eg_dummy(): lon_u_points = np.arange(0, 360, D_LON_N96, dtype="float32") lon_u_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LON_COORD_NAME, + coordname=um2nc.LONGITUDE, points=lon_u_points ) return lon_u_eg_dummy_coord @@ -772,7 +772,7 @@ def lat_standard_nd_dummy(): ), dtype="float32") lat_standard_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LAT_COORD_NAME, + coordname=um2nc.LATITUDE, points=lat_points ) return lat_standard_nd_dummy_coord @@ -785,7 +785,7 @@ def lon_standard_nd_dummy(): lon_points = np.arange(0, 360, D_LON_N96, dtype="float32") lon_standard_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LON_COORD_NAME, + coordname=um2nc.LONGITUDE, points=lon_points ) return lon_standard_nd_dummy_coord @@ -799,7 +799,7 @@ def lat_standard_eg_dummy(): D_LAT_N96, dtype="float32") lat_standard_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LAT_COORD_NAME, + coordname=um2nc.LATITUDE, points=lat_points ) return lat_standard_eg_dummy_coord @@ -811,7 +811,7 @@ def lon_standard_eg_dummy(): # grid from CM2 (which uses the End Game grid). lon_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") lon_standard_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LON_COORD_NAME, + coordname=um2nc.LONGITUDE, points=lon_points ) return lon_standard_eg_dummy_coord @@ -838,6 +838,17 @@ def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] +def assert_unmodified_coordinate(dummy_coord): + """ + Helper function to heck that a dummy_coordinate's attributes match + those expected for a coordinate that has not yet been modified + by fix_latlon_coords. + """ + assert dummy_coord.points.dtype == np.dtype("float32") + assert not dummy_coord.has_bounds() + assert dummy_coord.var_name is None + + # Tests of fix_latlon_coords. This function converts coordinate points # to double, adds bounds, and adds var_names to the coordinates. # The following tests check that these are done correctly. @@ -854,40 +865,29 @@ def test_fix_latlon_coords_river(ua_plev_cube, ) # Supplementary grid information used by fix_latlon_coords. - # Grid type, d_lat, d_lon don't matter for river grid, but must be set. - d_lat = D_LAT_N96 - d_lon = D_LON_N96 + # Grid type doesn't matter for river grid, but must be set. grid_type = um2nc.GRID_END_GAME - cube_lat_coord = cube_with_river_coords.coord(um2nc.LAT_COORD_NAME) - cube_lon_coord = cube_with_river_coords.coord(um2nc.LON_COORD_NAME) + cube_lat_coord = cube_with_river_coords.coord(um2nc.LATITUDE) + cube_lon_coord = cube_with_river_coords.coord(um2nc.LONGITUDE) # Checks prior to modifications. - assert cube_lat_coord.points.dtype == np.dtype("float32") - assert cube_lon_coord.points.dtype == np.dtype("float32") - - assert not cube_lat_coord.has_bounds() - assert not cube_lon_coord.has_bounds() - - assert cube_lat_coord.var_name is None - assert cube_lon_coord.var_name is None + assert_unmodified_coordinate(cube_lat_coord) + assert_unmodified_coordinate(cube_lon_coord) # Add mocks of guess_bounds method, to see that it is called. cube_lat_coord.guess_bounds = mock.Mock(return_value=None) cube_lon_coord.guess_bounds = mock.Mock(return_value=None) - um2nc.fix_latlon_coords(cube_with_river_coords, grid_type, d_lat, d_lon) - - expected_lon_name = "lon_river" - expected_lat_name = "lat_river" - expected_dtype = np.dtype("float64") + um2nc.fix_latlon_coords(cube_with_river_coords, grid_type, + D_LAT_N96, D_LON_N96) # Checks post modifications. - assert cube_lat_coord.var_name == expected_lat_name - assert cube_lon_coord.var_name == expected_lon_name + assert cube_lat_coord.var_name == um2nc.VAR_NAME_LAT_RIVER + assert cube_lon_coord.var_name == um2nc.VAR_NAME_LON_RIVER - assert cube_lat_coord.points.dtype == expected_dtype - assert cube_lon_coord.points.dtype == expected_dtype + assert cube_lat_coord.points.dtype == np.dtype("float64") + assert cube_lon_coord.points.dtype == np.dtype("float64") cube_lat_coord.guess_bounds.assert_called cube_lon_coord.guess_bounds.assert_called @@ -902,15 +902,6 @@ def test_fix_latlon_coords_uv(ua_plev_cube, Tests of the fix_lat_lon_coords for longitude u and latitude v coordinates on both the New Dynamics and End Game grids. """ - # Both coordinate sets are on an N96 grid. - d_lat = D_LAT_N96 - d_lon = D_LON_N96 - - # Expected values after modification - expected_lat_name = "lat_v" - expected_lon_name = "lon_u" - expected_dtype = np.dtype("float64") - coord_sets = [ (lat_v_nd_dummy, lon_u_nd_dummy, um2nc.GRID_NEW_DYNAMICS), (lat_v_eg_dummy, lon_u_eg_dummy, um2nc.GRID_END_GAME) @@ -924,27 +915,21 @@ def test_fix_latlon_coords_uv(ua_plev_cube, ) # Checks prior to modifications. - assert lat_coordinate.points.dtype == np.dtype("float32") - assert lon_coordinate.points.dtype == np.dtype("float32") - - assert not lat_coordinate.has_bounds() - assert not lon_coordinate.has_bounds() + assert_unmodified_coordinate(lat_coordinate) + assert_unmodified_coordinate(lon_coordinate) - assert lat_coordinate.var_name is None - assert lon_coordinate.var_name is None - - # Add mocks of guess_bounds method, to see that it is called. + # Add mocks of guess_bounds method, to check that it is called. lat_coordinate.guess_bounds = mock.Mock(return_value=None) lon_coordinate.guess_bounds = mock.Mock(return_value=None) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, - d_lat, d_lon) + D_LAT_N96, D_LON_N96) - assert lat_coordinate.var_name == expected_lat_name - assert lon_coordinate.var_name == expected_lon_name + assert lat_coordinate.var_name == um2nc.VAR_NAME_LAT_V + assert lon_coordinate.var_name == um2nc.VAR_NAME_LON_U - assert lat_coordinate.points.dtype == expected_dtype - assert lon_coordinate.points.dtype == expected_dtype + assert lat_coordinate.points.dtype == np.dtype("float64") + assert lon_coordinate.points.dtype == np.dtype("float64") lat_coordinate.guess_bounds.assert_called lon_coordinate.guess_bounds.assert_called @@ -960,15 +945,6 @@ def test_fix_latlon_coords_standard(ua_plev_cube, and latitude coordinates on both the New Dynamics and End Game grids. """ - # Both coordinate sets are on an N96 grid. - d_lat = D_LAT_N96 - d_lon = D_LON_N96 - - # Expected values after modification - expected_lat_name = "lat" - expected_lon_name = "lon" - expected_dtype = np.dtype("float64") - coord_sets = [ ( lat_standard_nd_dummy, @@ -990,27 +966,21 @@ def test_fix_latlon_coords_standard(ua_plev_cube, ) # Checks prior to modifications. - assert lat_coordinate.points.dtype == np.dtype("float32") - assert lon_coordinate.points.dtype == np.dtype("float32") - - assert not lat_coordinate.has_bounds() - assert not lon_coordinate.has_bounds() - - assert lat_coordinate.var_name is None - assert lon_coordinate.var_name is None + assert_unmodified_coordinate(lat_coordinate) + assert_unmodified_coordinate(lon_coordinate) - # Add mocks of guess_bounds method, to see that it is called. + # Add mocks of guess_bounds method, to check that it is called. lat_coordinate.guess_bounds = mock.Mock(return_value=None) lon_coordinate.guess_bounds = mock.Mock(return_value=None) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, - d_lat, d_lon) + D_LAT_N96, D_LON_N96) - assert lat_coordinate.var_name == expected_lat_name - assert lon_coordinate.var_name == expected_lon_name + assert lat_coordinate.var_name == um2nc.VAR_NAME_LAT_STANDARD + assert lon_coordinate.var_name == um2nc.VAR_NAME_LON_STANDARD - assert lat_coordinate.points.dtype == expected_dtype - assert lon_coordinate.points.dtype == expected_dtype + assert lat_coordinate.points.dtype == np.dtype("float64") + assert lon_coordinate.points.dtype == np.dtype("float64") lat_coordinate.guess_bounds.assert_called lon_coordinate.guess_bounds.assert_called @@ -1021,17 +991,15 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): Test that single point longitude and latitude coordinates are provided with global bounds. """ - d_lat = D_LAT_N96 - d_lon = D_LON_N96 grid_type = um2nc.GRID_NEW_DYNAMICS # Expected values after modification - expected_lat_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LAT_COORD_NAME] - expected_lon_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LON_COORD_NAME] + expected_lat_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LATITUDE] + expected_lon_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LONGITUDE] - lat_coord_single = DummyCoordinate(coordname=um2nc.LAT_COORD_NAME, + lat_coord_single = DummyCoordinate(coordname=um2nc.LATITUDE, points=np.array([0])) - lon_coord_single = DummyCoordinate(coordname=um2nc.LON_COORD_NAME, + lon_coord_single = DummyCoordinate(coordname=um2nc.LONGITUDE, points=np.array([0])) cube_with_uv_coords = DummyCubeWithCoords( @@ -1043,7 +1011,7 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): assert not lon_coord_single.has_bounds() um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, - d_lat, d_lon) + D_LAT_N96, D_LON_N96) assert lat_coord_single.has_bounds() assert lon_coord_single.has_bounds() @@ -1056,19 +1024,17 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): Test that existing coordinate bounds are not modified by fix_latlon_coords. """ - # Grid information required to run fix_latlon_coords - d_lat = D_LAT_N96 - d_lon = D_LON_N96 + # Grid information required to run fix_latlon_coords6 grid_type = um2nc.GRID_NEW_DYNAMICS # Expected values after modification lon_bounds = np.array([[0, 1]]) lat_bounds = np.array([[10, 25]]) - lat_coord = DummyCoordinate(coordname=um2nc.LAT_COORD_NAME, + lat_coord = DummyCoordinate(coordname=um2nc.LATITUDE, points=np.array([0]), bounds=lat_bounds.copy()) - lon_coord = DummyCoordinate(coordname=um2nc.LON_COORD_NAME, + lon_coord = DummyCoordinate(coordname=um2nc.LONGITUDE, points=np.array([0]), bounds=lon_bounds.copy()) @@ -1080,7 +1046,7 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): assert lon_coord.has_bounds() um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, - d_lat, d_lon) + D_LAT_N96, D_LON_N96) assert np.array_equal(lat_coord.bounds, lat_bounds) assert np.array_equal(lon_coord.bounds, lon_bounds) @@ -1098,8 +1064,6 @@ def _raise_CoordinateNotFoundError(coord_name): # Following values don't matter for test. Just needed as arguments grid_type = um2nc.GRID_NEW_DYNAMICS - dlat = 1.25 - dlon = 1.875 # Replace coord method to raise UnsupportedTimeSeriesError ua_plev_cube.coord = _raise_CoordinateNotFoundError @@ -1107,4 +1071,4 @@ def _raise_CoordinateNotFoundError(coord_name): with ( pytest.raises(um2nc.UnsupportedTimeSeriesError) ): - um2nc.fix_latlon_coords(ua_plev_cube, grid_type, dlat, dlon) + um2nc.fix_latlon_coords(ua_plev_cube, grid_type, D_LAT_N96, D_LON_N96) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index f356488..1bd37b4 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -40,18 +40,24 @@ # TODO: what is this limit & does it still exist? XCONV_LONG_NAME_LIMIT = 110 -LON_COORD_NAME = "longitude" -LAT_COORD_NAME = "latitude" +LONGITUDE = "longitude" +LATITUDE = "latitude" # Bounds for global single cells GLOBAL_COORD_BOUNDS = { - LON_COORD_NAME: np.array([[0., 360.]]), - LAT_COORD_NAME: np.array([[-90., 90.]]) + LONGITUDE: np.array([[0., 360.]]), + LATITUDE: np.array([[-90., 90.]]) } NUM_LAT_RIVER_GRID_POINTS = 180 NUM_LON_RIVER_GRID_POINTS = 360 +VAR_NAME_LAT_RIVER = "lat_river" +VAR_NAME_LON_RIVER = "lon_river" +VAR_NAME_LAT_V = "lat_v" +VAR_NAME_LON_U = "lon_u" +VAR_NAME_LAT_STANDARD = "lat" +VAR_NAME_LON_STANDARD = "lon" NC_FORMATS = { 1: 'NETCDF3_CLASSIC', @@ -135,18 +141,18 @@ def fix_lat_coord_name(lat_coordinate, grid_type, dlat): dlat: (float) meridional spacing between latitude grid points. """ - if lat_coordinate.name() != LAT_COORD_NAME: + if lat_coordinate.name() != LATITUDE: raise ValueError( f"Wrong coordinate {lat_coordinate.name()} supplied. " - f"Expected {LAT_COORD_NAME}." + f"Expected {LATITUDE}." ) if is_lat_river_grid(lat_coordinate.points): - lat_coordinate.var_name = 'lat_river' + lat_coordinate.var_name = VAR_NAME_LAT_RIVER elif is_lat_v_grid(lat_coordinate.points, grid_type, dlat): - lat_coordinate.var_name = 'lat_v' + lat_coordinate.var_name = VAR_NAME_LAT_V else: - lat_coordinate.var_name = 'lat' + lat_coordinate.var_name = VAR_NAME_LAT_STANDARD def fix_lon_coord_name(lon_coordinate, grid_type, dlon): @@ -164,18 +170,18 @@ def fix_lon_coord_name(lon_coordinate, grid_type, dlon): dlon: (float) zonal spacing between longitude grid points. """ - if lon_coordinate.name() != LON_COORD_NAME: + if lon_coordinate.name() != LONGITUDE: raise ValueError( f"Wrong coordinate {lon_coordinate.name()} supplied. " - f"Expected {LAT_COORD_NAME}." + f"Expected {LATITUDE}." ) if is_lon_river_grid(lon_coordinate.points): - lon_coordinate.var_name = 'lon_river' + lon_coordinate.var_name = VAR_NAME_LON_RIVER elif is_lon_u_grid(lon_coordinate.points, grid_type, dlon): - lon_coordinate.var_name = 'lon_u' + lon_coordinate.var_name = VAR_NAME_LON_U else: - lon_coordinate.var_name = 'lon' + lon_coordinate.var_name = VAR_NAME_LON_STANDARD def is_lat_river_grid(latitude_points): @@ -255,10 +261,10 @@ def add_latlon_coord_bounds(cube_coordinate): cube_coordinate: coordinate object from iris cube. """ coordinate_name = cube_coordinate.name() - if coordinate_name not in [LON_COORD_NAME, LAT_COORD_NAME]: + if coordinate_name not in [LONGITUDE, LATITUDE]: raise ValueError( f"Wrong coordinate {coordinate_name} supplied. " - f"Expected one of {LON_COORD_NAME}, {LAT_COORD_NAME}." + f"Expected one of {LONGITUDE}, {LATITUDE}." ) # Only add bounds if not already present. @@ -289,8 +295,8 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): """ try: - latitude_coordinate = cube.coord(LAT_COORD_NAME) - longitude_coordinate = cube.coord(LON_COORD_NAME) + latitude_coordinate = cube.coord(LATITUDE) + longitude_coordinate = cube.coord(LONGITUDE) except iris.exceptions.CoordinateNotFoundError: msg = ( "Missing latitude or longitude coordinate for variable (possible timeseries?): \n" From c1ecfdc91eace4c25f96a198f379f326ab67f826 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Wed, 28 Aug 2024 14:11:49 +1000 Subject: [PATCH 18/27] Replace dummycoords with iris DimCoords --- test/test_um2netcdf.py | 236 +++++++++++++++++------------------------ 1 file changed, 100 insertions(+), 136 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 1d27716..3059dc5 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -12,6 +12,7 @@ import mule import mule.ff import iris.cube +import iris.coords D_LAT_N96 = 1.25 D_LON_N96 = 1.875 @@ -661,109 +662,84 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units -@dataclass -class DummyCoordinate: - """ - Imitation cube coordinate for unit testing. - Includes additional information that is not part of - a standard iris cube coordinate, but is useful for testing, - including MuleVars grid metadata, and atribute values expected - after testing. - """ - coordname: str - points: np.ndarray - bounds: np.ndarray = None - # Note that var_name attribute is different thing to return - # value of name() method. - var_name: str = None - - def name(self): - return self.coordname - - def has_bounds(self): - return self.bounds is not None - - @pytest.fixture -def lat_river_dummy(): - # Dummy coordinate imitating UM V7.3s river grid. Must have length 180. +def lat_river(): + # iris DimCoord imitating UM V7.3s river grid. Must have length 180. lat_river_points = np.arange(-90., 90, dtype="float32") + 0.5 - - lat_river_dummy_coord = DummyCoordinate( - coordname=um2nc.LATITUDE, - points=lat_river_points + lat_river_coord = iris.coords.DimCoord( + points=lat_river_points, + standard_name=um2nc.LATITUDE ) - return lat_river_dummy_coord + return lat_river_coord @pytest.fixture -def lon_river_dummy(): - # Dummy coordinate imitating UM V7.3s river grid. Must have length 90. +def lon_river(): + # iris DimCoord imitating UM V7.3s river grid. Must have length 90. lon_river_points = np.arange(0., 360., dtype="float32") + 0.5 - - lon_river_dummy_coord = DummyCoordinate( - coordname=um2nc.LONGITUDE, + lon_river_coord = iris.coords.DimCoord( points=lon_river_points, + standard_name=um2nc.LONGITUDE ) - return lon_river_dummy_coord + return lon_river_coord @pytest.fixture -def lat_v_nd_dummy(): - # Dummy coordinate imitating imitating the real +def lat_v_nd(): + # iris DimCoord imitating imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). lat_v_points = np.arange(-90.+0.5*D_LAT_N96, 90, D_LAT_N96, dtype="float32") - lat_v_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LATITUDE, + lat_v_nd_coord = iris.coords.DimCoord( points=lat_v_points, + standard_name=um2nc.LATITUDE, ) - return lat_v_nd_dummy_coord + return lat_v_nd_coord @pytest.fixture -def lon_u_nd_dummy(): - # Dummy coordinate imitating the real +def lon_u_nd(): + # iris DimCoord imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). lon_u_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") - lon_u_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LONGITUDE, + lon_u_nd_coord = iris.coords.DimCoord( points=lon_u_points, - ) - return lon_u_nd_dummy_coord + standard_name=um2nc.LONGITUDE, + ) + return lon_u_nd_coord @pytest.fixture -def lat_v_eg_dummy(): - # Dummy coordinate imitating the real +def lat_v_eg(): + # iris DimCoord imitating the real # lat_v grid from CM2 (which uses the End Game grid). lat_v_points = np.arange(-90., 91, D_LAT_N96, dtype="float32") - lat_v_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LATITUDE, - points=lat_v_points + lat_v_eg_coord = iris.coords.DimCoord( + points=lat_v_points, + standard_name=um2nc.LATITUDE ) - return lat_v_eg_dummy_coord + return lat_v_eg_coord @pytest.fixture -def lon_u_eg_dummy(): - # Dummy coordinate imitating the real +def lon_u_eg(): + # iris DimCoord imitating the real # lon_v grid from CM2 (which uses the End Game grid). lon_u_points = np.arange(0, 360, D_LON_N96, dtype="float32") - lon_u_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LONGITUDE, - points=lon_u_points + lon_u_eg_coord = iris.coords.DimCoord( + points=lon_u_points, + standard_name=um2nc.LONGITUDE ) - return lon_u_eg_dummy_coord + return lon_u_eg_coord @pytest.fixture -def lat_standard_nd_dummy(): - # Dummy coordinate imitating the standard latitude +def lat_standard_nd(): + # iris DimCoord imitating the standard latitude # grid from ESM1.5 (which uses the New Dynamics grid). lat_points_middle = np.arange(-88.75, 89., D_LAT_N96) lat_points = np.concatenate(([-90], @@ -771,55 +747,55 @@ def lat_standard_nd_dummy(): [90.] ), dtype="float32") - lat_standard_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LATITUDE, - points=lat_points + lat_standard_nd_coord = iris.coords.DimCoord( + points=lat_points, + standard_name=um2nc.LATITUDE ) - return lat_standard_nd_dummy_coord + return lat_standard_nd_coord @pytest.fixture -def lon_standard_nd_dummy(): - # Dummy coordinate imitating the standard longitude +def lon_standard_nd(): + # iris DimCoord imitating the standard longitude # grid from ESM1.5 (which uses the New Dynamics grid). lon_points = np.arange(0, 360, D_LON_N96, dtype="float32") - lon_standard_nd_dummy_coord = DummyCoordinate( - coordname=um2nc.LONGITUDE, - points=lon_points + lon_standard_nd_coord = iris.coords.DimCoord( + points=lon_points, + standard_name=um2nc.LONGITUDE ) - return lon_standard_nd_dummy_coord + return lon_standard_nd_coord @pytest.fixture -def lat_standard_eg_dummy(): - # Dummy coordinate imitating the standard latitude +def lat_standard_eg(): + # iris DimCoord imitating the standard latitude # grid from CM2 (which uses the End Game grid). lat_points = np.arange(-90 + 0.5*D_LAT_N96, 90., D_LAT_N96, dtype="float32") - lat_standard_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LATITUDE, - points=lat_points + lat_standard_eg_coord = iris.coords.DimCoord( + points=lat_points, + standard_name=um2nc.LATITUDE ) - return lat_standard_eg_dummy_coord + return lat_standard_eg_coord @pytest.fixture -def lon_standard_eg_dummy(): - # Dummy coordinate imitating the standard longitude +def lon_standard_eg(): + # iris DimCoord imitating the standard longitude # grid from CM2 (which uses the End Game grid). lon_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") - lon_standard_eg_dummy_coord = DummyCoordinate( - coordname=um2nc.LONGITUDE, - points=lon_points + lon_standard_eg_coord = iris.coords.DimCoord( + points=lon_points, + standard_name=um2nc.LONGITUDE ) - return lon_standard_eg_dummy_coord + return lon_standard_eg_coord class DummyCubeWithCoords(DummyCube): # DummyCube with coordinates, which can be filled with - # DummyCoordinate objects for testing. + # iris.DimCoord objects for testing. def __init__(self, dummy_cube, coords=[]): super().__init__(dummy_cube.item_code, dummy_cube.var_name, @@ -831,37 +807,37 @@ def __init__(self, dummy_cube, coords=[]): # access a coordinate via the coord method matches the # coordinate's name. self.coordinate_dict = {} - for dummy_coord in coords: - self.coordinate_dict[dummy_coord.name()] = dummy_coord + for coord in coords: + self.coordinate_dict[coord.name()] = coord def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] -def assert_unmodified_coordinate(dummy_coord): +def assert_unmodified_coordinate(coord): """ - Helper function to heck that a dummy_coordinate's attributes match + Helper function to check that a coordinate's attributes match those expected for a coordinate that has not yet been modified by fix_latlon_coords. """ - assert dummy_coord.points.dtype == np.dtype("float32") - assert not dummy_coord.has_bounds() - assert dummy_coord.var_name is None + assert coord.points.dtype == np.dtype("float32") + assert not coord.has_bounds() + assert coord.var_name is None # Tests of fix_latlon_coords. This function converts coordinate points # to double, adds bounds, and adds var_names to the coordinates. # The following tests check that these are done correctly. def test_fix_latlon_coords_river(ua_plev_cube, - lat_river_dummy, - lon_river_dummy): + lat_river, + lon_river): """ Tests of the fix_lat_lon_coords function on river grid coordinates. """ cube_with_river_coords = DummyCubeWithCoords( dummy_cube=ua_plev_cube, - coords=[lat_river_dummy, lon_river_dummy] + coords=[lat_river, lon_river] ) # Supplementary grid information used by fix_latlon_coords. @@ -875,10 +851,6 @@ def test_fix_latlon_coords_river(ua_plev_cube, assert_unmodified_coordinate(cube_lat_coord) assert_unmodified_coordinate(cube_lon_coord) - # Add mocks of guess_bounds method, to see that it is called. - cube_lat_coord.guess_bounds = mock.Mock(return_value=None) - cube_lon_coord.guess_bounds = mock.Mock(return_value=None) - um2nc.fix_latlon_coords(cube_with_river_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -889,22 +861,22 @@ def test_fix_latlon_coords_river(ua_plev_cube, assert cube_lat_coord.points.dtype == np.dtype("float64") assert cube_lon_coord.points.dtype == np.dtype("float64") - cube_lat_coord.guess_bounds.assert_called - cube_lon_coord.guess_bounds.assert_called + assert cube_lat_coord.has_bounds() + assert cube_lon_coord.has_bounds() def test_fix_latlon_coords_uv(ua_plev_cube, - lat_v_nd_dummy, - lon_u_nd_dummy, - lat_v_eg_dummy, - lon_u_eg_dummy): + lat_v_nd, + lon_u_nd, + lat_v_eg, + lon_u_eg): """ Tests of the fix_lat_lon_coords for longitude u and latitude v coordinates on both the New Dynamics and End Game grids. """ coord_sets = [ - (lat_v_nd_dummy, lon_u_nd_dummy, um2nc.GRID_NEW_DYNAMICS), - (lat_v_eg_dummy, lon_u_eg_dummy, um2nc.GRID_END_GAME) + (lat_v_nd, lon_u_nd, um2nc.GRID_NEW_DYNAMICS), + (lat_v_eg, lon_u_eg, um2nc.GRID_END_GAME) ] for lat_coordinate, lon_coordinate, grid_type in coord_sets: @@ -918,10 +890,6 @@ def test_fix_latlon_coords_uv(ua_plev_cube, assert_unmodified_coordinate(lat_coordinate) assert_unmodified_coordinate(lon_coordinate) - # Add mocks of guess_bounds method, to check that it is called. - lat_coordinate.guess_bounds = mock.Mock(return_value=None) - lon_coordinate.guess_bounds = mock.Mock(return_value=None) - um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -931,15 +899,15 @@ def test_fix_latlon_coords_uv(ua_plev_cube, assert lat_coordinate.points.dtype == np.dtype("float64") assert lon_coordinate.points.dtype == np.dtype("float64") - lat_coordinate.guess_bounds.assert_called - lon_coordinate.guess_bounds.assert_called + assert lat_coordinate.has_bounds() + assert lat_coordinate.has_bounds() def test_fix_latlon_coords_standard(ua_plev_cube, - lat_standard_nd_dummy, - lon_standard_nd_dummy, - lat_standard_eg_dummy, - lon_standard_eg_dummy): + lat_standard_nd, + lon_standard_nd, + lat_standard_eg, + lon_standard_eg): """ Tests of the fix_lat_lon_coords for standard longitude and latitude coordinates on both the New Dynamics and @@ -947,13 +915,13 @@ def test_fix_latlon_coords_standard(ua_plev_cube, """ coord_sets = [ ( - lat_standard_nd_dummy, - lon_standard_nd_dummy, + lat_standard_nd, + lon_standard_nd, um2nc.GRID_NEW_DYNAMICS ), ( - lat_standard_eg_dummy, - lon_standard_eg_dummy, + lat_standard_eg, + lon_standard_eg, um2nc.GRID_END_GAME ) ] @@ -969,10 +937,6 @@ def test_fix_latlon_coords_standard(ua_plev_cube, assert_unmodified_coordinate(lat_coordinate) assert_unmodified_coordinate(lon_coordinate) - # Add mocks of guess_bounds method, to check that it is called. - lat_coordinate.guess_bounds = mock.Mock(return_value=None) - lon_coordinate.guess_bounds = mock.Mock(return_value=None) - um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -982,8 +946,8 @@ def test_fix_latlon_coords_standard(ua_plev_cube, assert lat_coordinate.points.dtype == np.dtype("float64") assert lon_coordinate.points.dtype == np.dtype("float64") - lat_coordinate.guess_bounds.assert_called - lon_coordinate.guess_bounds.assert_called + assert lat_coordinate.has_bounds() + assert lat_coordinate.has_bounds() def test_fix_latlon_coords_single_point(ua_plev_cube): @@ -997,10 +961,10 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): expected_lat_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LATITUDE] expected_lon_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LONGITUDE] - lat_coord_single = DummyCoordinate(coordname=um2nc.LATITUDE, - points=np.array([0])) - lon_coord_single = DummyCoordinate(coordname=um2nc.LONGITUDE, - points=np.array([0])) + lat_coord_single = iris.coords.DimCoord(points=np.array([0]), + standard_name=um2nc.LATITUDE) + lon_coord_single = iris.coords.DimCoord(points=np.array([0]), + standard_name=um2nc.LONGITUDE) cube_with_uv_coords = DummyCubeWithCoords( dummy_cube=ua_plev_cube, @@ -1031,12 +995,12 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): lon_bounds = np.array([[0, 1]]) lat_bounds = np.array([[10, 25]]) - lat_coord = DummyCoordinate(coordname=um2nc.LATITUDE, - points=np.array([0]), - bounds=lat_bounds.copy()) - lon_coord = DummyCoordinate(coordname=um2nc.LONGITUDE, - points=np.array([0]), - bounds=lon_bounds.copy()) + lat_coord = iris.coords.DimCoord(points=np.array([0]), + standard_name=um2nc.LATITUDE, + bounds=lat_bounds.copy()) + lon_coord = iris.coords.DimCoord(points=np.array([0]), + standard_name=um2nc.LONGITUDE, + bounds=lon_bounds.copy()) cube_with_uv_coords = DummyCubeWithCoords( dummy_cube=ua_plev_cube, From c6f67538003185d6b2fe08fdfab5096aa75e9f8d Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Wed, 28 Aug 2024 14:33:53 +1000 Subject: [PATCH 19/27] Replace repeated asserts with helper functions --- test/test_um2netcdf.py | 57 +++++++++++++++++++++--------------------- umpost/um2netcdf.py | 8 +----- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index b4b1e5e..026c373 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -814,15 +814,26 @@ def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] -def assert_unmodified_coordinate(coord): +def assert_unmodified_coordinates(lat_coord, lon_coord): """ Helper function to check that a coordinate's attributes match those expected for a coordinate that has not yet been modified by fix_latlon_coords. """ - assert coord.points.dtype == np.dtype("float32") - assert not coord.has_bounds() - assert coord.var_name is None + for coord in [lat_coord, lon_coord]: + assert coord.points.dtype == np.dtype("float32") + assert not coord.has_bounds() + assert coord.var_name is None + + +def assert_dtype_float64(lat_coord, lon_coord): + assert lat_coord.points.dtype == np.dtype("float64") + assert lon_coord.points.dtype == np.dtype("float64") + + +def assert_has_bounds(lat_coord, lon_coord): + assert lat_coord.has_bounds() + assert lon_coord.has_bounds() # Tests of fix_latlon_coords. This function converts coordinate points @@ -848,8 +859,7 @@ def test_fix_latlon_coords_river(ua_plev_cube, cube_lon_coord = cube_with_river_coords.coord(um2nc.LONGITUDE) # Checks prior to modifications. - assert_unmodified_coordinate(cube_lat_coord) - assert_unmodified_coordinate(cube_lon_coord) + assert_unmodified_coordinates(cube_lat_coord, cube_lon_coord) um2nc.fix_latlon_coords(cube_with_river_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -858,11 +868,8 @@ def test_fix_latlon_coords_river(ua_plev_cube, assert cube_lat_coord.var_name == um2nc.VAR_NAME_LAT_RIVER assert cube_lon_coord.var_name == um2nc.VAR_NAME_LON_RIVER - assert cube_lat_coord.points.dtype == np.dtype("float64") - assert cube_lon_coord.points.dtype == np.dtype("float64") - - assert cube_lat_coord.has_bounds() - assert cube_lon_coord.has_bounds() + assert_dtype_float64(cube_lat_coord, cube_lon_coord) + assert_has_bounds(cube_lat_coord, cube_lon_coord) def test_fix_latlon_coords_uv(ua_plev_cube, @@ -887,8 +894,7 @@ def test_fix_latlon_coords_uv(ua_plev_cube, ) # Checks prior to modifications. - assert_unmodified_coordinate(lat_coordinate) - assert_unmodified_coordinate(lon_coordinate) + assert_unmodified_coordinates(lat_coordinate, lon_coordinate) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -896,11 +902,8 @@ def test_fix_latlon_coords_uv(ua_plev_cube, assert lat_coordinate.var_name == um2nc.VAR_NAME_LAT_V assert lon_coordinate.var_name == um2nc.VAR_NAME_LON_U - assert lat_coordinate.points.dtype == np.dtype("float64") - assert lon_coordinate.points.dtype == np.dtype("float64") - - assert lat_coordinate.has_bounds() - assert lat_coordinate.has_bounds() + assert_dtype_float64(lat_coordinate, lon_coordinate) + assert_has_bounds(lat_coordinate, lon_coordinate) def test_fix_latlon_coords_standard(ua_plev_cube, @@ -934,8 +937,7 @@ def test_fix_latlon_coords_standard(ua_plev_cube, ) # Checks prior to modifications. - assert_unmodified_coordinate(lat_coordinate) - assert_unmodified_coordinate(lon_coordinate) + assert_unmodified_coordinates(lat_coordinate, lon_coordinate) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -943,11 +945,8 @@ def test_fix_latlon_coords_standard(ua_plev_cube, assert lat_coordinate.var_name == um2nc.VAR_NAME_LAT_STANDARD assert lon_coordinate.var_name == um2nc.VAR_NAME_LON_STANDARD - assert lat_coordinate.points.dtype == np.dtype("float64") - assert lon_coordinate.points.dtype == np.dtype("float64") - - assert lat_coordinate.has_bounds() - assert lat_coordinate.has_bounds() + assert_dtype_float64(lat_coordinate, lon_coordinate) + assert_has_bounds(lat_coordinate, lon_coordinate) def test_fix_latlon_coords_single_point(ua_plev_cube): @@ -977,8 +976,7 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) - assert lat_coord_single.has_bounds() - assert lon_coord_single.has_bounds() + assert_has_bounds(lat_coord_single, lon_coord_single) assert np.array_equal(lat_coord_single.bounds, expected_lat_bounds) assert np.array_equal(lon_coord_single.bounds, expected_lon_bounds) @@ -1006,8 +1004,7 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): dummy_cube=ua_plev_cube, coords=[lat_coord, lon_coord] ) - assert lat_coord.has_bounds() - assert lon_coord.has_bounds() + assert_has_bounds(lat_coord, lon_coord) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -1036,6 +1033,8 @@ def _raise_CoordinateNotFoundError(coord_name): pytest.raises(um2nc.UnsupportedTimeSeriesError) ): um2nc.fix_latlon_coords(ua_plev_cube, grid_type, D_LAT_N96, D_LON_N96) + + def test_fix_cell_methods_drop_hours(): # ensure cell methods with "hour" in the interval name are translated to # empty intervals diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index d901c68..8d39d7b 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -524,13 +524,7 @@ def process(infile, outfile, args): # Interval in cell methods isn't reliable so better to remove it. c.cell_methods = fix_cell_methods(c.cell_methods) - try: - fix_latlon_coord(c, mv.grid_type, mv.d_lat, mv.d_lon) - except iris.exceptions.CoordinateNotFoundError: - print('\nMissing lat/lon coordinates for variable (possible timeseries?)\n') - print(c) - raise Exception("Variable can not be processed") - + fix_latlon_coords(c, mv.grid_type, mv.d_lat, mv.d_lon) fix_level_coord(c, mv.z_rho, mv.z_theta) if do_masking: From 171b7ffa6ed52a699b72fa35fe8cc312ca335923 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 30 Aug 2024 11:07:33 +1000 Subject: [PATCH 20/27] Simplify coordinate fixtures using decorators --- test/test_um2netcdf.py | 146 +++++++++++++++++------------------------ 1 file changed, 61 insertions(+), 85 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 026c373..34c6004 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -662,83 +662,74 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units +def to_iris_DimCoord(points_and_name_func): + + def DimCoord_maker(): + points, name = points_and_name_func() + return iris.coords.DimCoord( + points=points, + standard_name=name + ) + + return DimCoord_maker + + @pytest.fixture -def lat_river(): +@to_iris_DimCoord +def lat_river_coord(): # iris DimCoord imitating UM V7.3s river grid. Must have length 180. lat_river_points = np.arange(-90., 90, dtype="float32") + 0.5 - lat_river_coord = iris.coords.DimCoord( - points=lat_river_points, - standard_name=um2nc.LATITUDE - ) - return lat_river_coord + return lat_river_points, um2nc.LATITUDE @pytest.fixture -def lon_river(): +@to_iris_DimCoord +def lon_river_coord(): # iris DimCoord imitating UM V7.3s river grid. Must have length 90. lon_river_points = np.arange(0., 360., dtype="float32") + 0.5 - lon_river_coord = iris.coords.DimCoord( - points=lon_river_points, - standard_name=um2nc.LONGITUDE - ) - return lon_river_coord + return lon_river_points, um2nc.LONGITUDE @pytest.fixture -def lat_v_nd(): - # iris DimCoord imitating imitating the real +@to_iris_DimCoord +def lat_v_nd_coord(): + # iris DimCoord imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). lat_v_points = np.arange(-90.+0.5*D_LAT_N96, 90, D_LAT_N96, dtype="float32") - - lat_v_nd_coord = iris.coords.DimCoord( - points=lat_v_points, - standard_name=um2nc.LATITUDE, - ) - return lat_v_nd_coord + return lat_v_points, um2nc.LATITUDE @pytest.fixture -def lon_u_nd(): +@to_iris_DimCoord +def lon_u_nd_coord(): # iris DimCoord imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). lon_u_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") - - lon_u_nd_coord = iris.coords.DimCoord( - points=lon_u_points, - standard_name=um2nc.LONGITUDE, - ) - return lon_u_nd_coord + return lon_u_points, um2nc.LONGITUDE @pytest.fixture -def lat_v_eg(): +@to_iris_DimCoord +def lat_v_eg_coord(): # iris DimCoord imitating the real # lat_v grid from CM2 (which uses the End Game grid). lat_v_points = np.arange(-90., 91, D_LAT_N96, dtype="float32") - - lat_v_eg_coord = iris.coords.DimCoord( - points=lat_v_points, - standard_name=um2nc.LATITUDE - ) - return lat_v_eg_coord + return lat_v_points, um2nc.LATITUDE @pytest.fixture -def lon_u_eg(): +@to_iris_DimCoord +def lon_u_eg_coord(): # iris DimCoord imitating the real # lon_v grid from CM2 (which uses the End Game grid). lon_u_points = np.arange(0, 360, D_LON_N96, dtype="float32") - - lon_u_eg_coord = iris.coords.DimCoord( - points=lon_u_points, - standard_name=um2nc.LONGITUDE - ) - return lon_u_eg_coord + return lon_u_points, um2nc.LONGITUDE @pytest.fixture -def lat_standard_nd(): +@to_iris_DimCoord +def lat_standard_nd_coord(): # iris DimCoord imitating the standard latitude # grid from ESM1.5 (which uses the New Dynamics grid). lat_points_middle = np.arange(-88.75, 89., D_LAT_N96) @@ -746,51 +737,35 @@ def lat_standard_nd(): lat_points_middle, [90.] ), dtype="float32") - - lat_standard_nd_coord = iris.coords.DimCoord( - points=lat_points, - standard_name=um2nc.LATITUDE - ) - return lat_standard_nd_coord + return lat_points, um2nc.LATITUDE @pytest.fixture -def lon_standard_nd(): +@to_iris_DimCoord +def lon_standard_nd_coord(): # iris DimCoord imitating the standard longitude # grid from ESM1.5 (which uses the New Dynamics grid). lon_points = np.arange(0, 360, D_LON_N96, dtype="float32") - - lon_standard_nd_coord = iris.coords.DimCoord( - points=lon_points, - standard_name=um2nc.LONGITUDE - ) - return lon_standard_nd_coord + return lon_points, um2nc.LONGITUDE @pytest.fixture -def lat_standard_eg(): +@to_iris_DimCoord +def lat_standard_eg_coord(): # iris DimCoord imitating the standard latitude # grid from CM2 (which uses the End Game grid). lat_points = np.arange(-90 + 0.5*D_LAT_N96, 90., D_LAT_N96, dtype="float32") - - lat_standard_eg_coord = iris.coords.DimCoord( - points=lat_points, - standard_name=um2nc.LATITUDE - ) - return lat_standard_eg_coord + return lat_points, um2nc.LATITUDE @pytest.fixture -def lon_standard_eg(): +@to_iris_DimCoord +def lon_standard_eg_coord(): # iris DimCoord imitating the standard longitude # grid from CM2 (which uses the End Game grid). lon_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") - lon_standard_eg_coord = iris.coords.DimCoord( - points=lon_points, - standard_name=um2nc.LONGITUDE - ) - return lon_standard_eg_coord + return lon_points, um2nc.LONGITUDE class DummyCubeWithCoords(DummyCube): @@ -840,15 +815,15 @@ def assert_has_bounds(lat_coord, lon_coord): # to double, adds bounds, and adds var_names to the coordinates. # The following tests check that these are done correctly. def test_fix_latlon_coords_river(ua_plev_cube, - lat_river, - lon_river): + lat_river_coord, + lon_river_coord): """ Tests of the fix_lat_lon_coords function on river grid coordinates. """ cube_with_river_coords = DummyCubeWithCoords( dummy_cube=ua_plev_cube, - coords=[lat_river, lon_river] + coords=[lat_river_coord, lon_river_coord] ) # Supplementary grid information used by fix_latlon_coords. @@ -873,17 +848,18 @@ def test_fix_latlon_coords_river(ua_plev_cube, def test_fix_latlon_coords_uv(ua_plev_cube, - lat_v_nd, - lon_u_nd, - lat_v_eg, - lon_u_eg): + lat_v_nd_coord, + lon_u_nd_coord, + lat_v_eg_coord, + lon_u_eg_coord, + ): """ Tests of the fix_lat_lon_coords for longitude u and latitude v coordinates on both the New Dynamics and End Game grids. """ coord_sets = [ - (lat_v_nd, lon_u_nd, um2nc.GRID_NEW_DYNAMICS), - (lat_v_eg, lon_u_eg, um2nc.GRID_END_GAME) + (lat_v_nd_coord, lon_u_nd_coord, um2nc.GRID_NEW_DYNAMICS), + (lat_v_eg_coord, lon_u_eg_coord, um2nc.GRID_END_GAME) ] for lat_coordinate, lon_coordinate, grid_type in coord_sets: @@ -907,10 +883,10 @@ def test_fix_latlon_coords_uv(ua_plev_cube, def test_fix_latlon_coords_standard(ua_plev_cube, - lat_standard_nd, - lon_standard_nd, - lat_standard_eg, - lon_standard_eg): + lat_standard_nd_coord, + lon_standard_nd_coord, + lat_standard_eg_coord, + lon_standard_eg_coord): """ Tests of the fix_lat_lon_coords for standard longitude and latitude coordinates on both the New Dynamics and @@ -918,13 +894,13 @@ def test_fix_latlon_coords_standard(ua_plev_cube, """ coord_sets = [ ( - lat_standard_nd, - lon_standard_nd, + lat_standard_nd_coord, + lon_standard_nd_coord, um2nc.GRID_NEW_DYNAMICS ), ( - lat_standard_eg, - lon_standard_eg, + lat_standard_eg_coord, + lon_standard_eg_coord, um2nc.GRID_END_GAME ) ] From 05f35f2ef385b1237a953688d74927360c288bbb Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 30 Aug 2024 14:38:41 +1000 Subject: [PATCH 21/27] Cleanup and simplification from review --- test/test_um2netcdf.py | 65 +++++++++++++++++++++++------------------- umpost/um2netcdf.py | 4 +-- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 34c6004..cf92a5f 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -408,6 +408,13 @@ def __init__(self, item_code, var_name=None, attributes=None, units=None): # tests. Would cause a KeyError anyway? # self.coord = {} + + def coord(self): + raise NotImplementedError( + "coord() method not implemented on DummyCube" + ) + + def name(self): # mimic iris API return self.var_name @@ -677,7 +684,8 @@ def DimCoord_maker(): @pytest.fixture @to_iris_DimCoord def lat_river_coord(): - # iris DimCoord imitating UM V7.3s river grid. Must have length 180. + # iris DimCoord imitating UM V7.3s 1x1 degree river grid. + # Must have length 180. lat_river_points = np.arange(-90., 90, dtype="float32") + 0.5 return lat_river_points, um2nc.LATITUDE @@ -685,7 +693,8 @@ def lat_river_coord(): @pytest.fixture @to_iris_DimCoord def lon_river_coord(): - # iris DimCoord imitating UM V7.3s river grid. Must have length 90. + # iris DimCoord imitating UM V7.3s 1x1 degree river grid. + # Must have length 360. lon_river_points = np.arange(0., 360., dtype="float32") + 0.5 return lon_river_points, um2nc.LONGITUDE @@ -695,6 +704,8 @@ def lon_river_coord(): def lat_v_nd_coord(): # iris DimCoord imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). + # This grid is offset half a grid cell compared to the standard + # New Dynamics latitude grid. lat_v_points = np.arange(-90.+0.5*D_LAT_N96, 90, D_LAT_N96, dtype="float32") return lat_v_points, um2nc.LATITUDE @@ -705,6 +716,8 @@ def lat_v_nd_coord(): def lon_u_nd_coord(): # iris DimCoord imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). + # This grid is offset half a grid cell compared to the standard + # New Dynamics longitude grid. lon_u_points = np.arange(0.5*D_LON_N96, 360, D_LON_N96, dtype="float32") return lon_u_points, um2nc.LONGITUDE @@ -714,6 +727,8 @@ def lon_u_nd_coord(): def lat_v_eg_coord(): # iris DimCoord imitating the real # lat_v grid from CM2 (which uses the End Game grid). + # This grid is offset half a grid cell compared to the standard + # End Game latitude grid. lat_v_points = np.arange(-90., 91, D_LAT_N96, dtype="float32") return lat_v_points, um2nc.LATITUDE @@ -723,6 +738,8 @@ def lat_v_eg_coord(): def lon_u_eg_coord(): # iris DimCoord imitating the real # lon_v grid from CM2 (which uses the End Game grid). + # This grid is offset half a grid cell compared to the standard + # New Dynamics longitude grid. lon_u_points = np.arange(0, 360, D_LON_N96, dtype="float32") return lon_u_points, um2nc.LONGITUDE @@ -732,11 +749,7 @@ def lon_u_eg_coord(): def lat_standard_nd_coord(): # iris DimCoord imitating the standard latitude # grid from ESM1.5 (which uses the New Dynamics grid). - lat_points_middle = np.arange(-88.75, 89., D_LAT_N96) - lat_points = np.concatenate(([-90], - lat_points_middle, - [90.] - ), dtype="float32") + lat_points = np.arange(-90, 91, D_LAT_N96, dtype="float32") return lat_points, um2nc.LATITUDE @@ -771,7 +784,7 @@ def lon_standard_eg_coord(): class DummyCubeWithCoords(DummyCube): # DummyCube with coordinates, which can be filled with # iris.DimCoord objects for testing. - def __init__(self, dummy_cube, coords=[]): + def __init__(self, dummy_cube, coords=None): super().__init__(dummy_cube.item_code, dummy_cube.var_name, dummy_cube.attributes, @@ -781,15 +794,18 @@ def __init__(self, dummy_cube, coords=[]): # given by the coordinate names. This ensures that the key used to # access a coordinate via the coord method matches the # coordinate's name. - self.coordinate_dict = {} - for coord in coords: - self.coordinate_dict[coord.name()] = coord + if coords is not None: + self.coordinate_dict = { + coord.name(): coord for coord in coords + } + else: + self.coordinate_dict = {} def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] -def assert_unmodified_coordinates(lat_coord, lon_coord): +def assert_coordinates_are_unmodified(lat_coord, lon_coord): """ Helper function to check that a coordinate's attributes match those expected for a coordinate that has not yet been modified @@ -826,17 +842,13 @@ def test_fix_latlon_coords_river(ua_plev_cube, coords=[lat_river_coord, lon_river_coord] ) - # Supplementary grid information used by fix_latlon_coords. - # Grid type doesn't matter for river grid, but must be set. - grid_type = um2nc.GRID_END_GAME - cube_lat_coord = cube_with_river_coords.coord(um2nc.LATITUDE) cube_lon_coord = cube_with_river_coords.coord(um2nc.LONGITUDE) # Checks prior to modifications. - assert_unmodified_coordinates(cube_lat_coord, cube_lon_coord) + assert_coordinates_are_unmodified(cube_lat_coord, cube_lon_coord) - um2nc.fix_latlon_coords(cube_with_river_coords, grid_type, + um2nc.fix_latlon_coords(cube_with_river_coords, um2nc.GRID_END_GAME, D_LAT_N96, D_LON_N96) # Checks post modifications. @@ -870,7 +882,7 @@ def test_fix_latlon_coords_uv(ua_plev_cube, ) # Checks prior to modifications. - assert_unmodified_coordinates(lat_coordinate, lon_coordinate) + assert_coordinates_are_unmodified(lat_coordinate, lon_coordinate) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -913,7 +925,7 @@ def test_fix_latlon_coords_standard(ua_plev_cube, ) # Checks prior to modifications. - assert_unmodified_coordinates(lat_coordinate, lon_coordinate) + assert_coordinates_are_unmodified(lat_coordinate, lon_coordinate) um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, D_LAT_N96, D_LON_N96) @@ -930,7 +942,6 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): Test that single point longitude and latitude coordinates are provided with global bounds. """ - grid_type = um2nc.GRID_NEW_DYNAMICS # Expected values after modification expected_lat_bounds = um2nc.GLOBAL_COORD_BOUNDS[um2nc.LATITUDE] @@ -949,7 +960,7 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): assert not lat_coord_single.has_bounds() assert not lon_coord_single.has_bounds() - um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, + um2nc.fix_latlon_coords(cube_with_uv_coords, um2nc.GRID_NEW_DYNAMICS, D_LAT_N96, D_LON_N96) assert_has_bounds(lat_coord_single, lon_coord_single) @@ -962,8 +973,6 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): Test that existing coordinate bounds are not modified by fix_latlon_coords. """ - # Grid information required to run fix_latlon_coords6 - grid_type = um2nc.GRID_NEW_DYNAMICS # Expected values after modification lon_bounds = np.array([[0, 1]]) @@ -982,7 +991,7 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): ) assert_has_bounds(lat_coord, lon_coord) - um2nc.fix_latlon_coords(cube_with_uv_coords, grid_type, + um2nc.fix_latlon_coords(cube_with_uv_coords, um2nc.GRID_NEW_DYNAMICS, D_LAT_N96, D_LON_N96) assert np.array_equal(lat_coord.bounds, lat_bounds) @@ -999,16 +1008,14 @@ def _raise_CoordinateNotFoundError(coord_name): # Iris cube's ".coord" method raise iris.exceptions.CoordinateNotFoundError(coord_name) - # Following values don't matter for test. Just needed as arguments - grid_type = um2nc.GRID_NEW_DYNAMICS - # Replace coord method to raise UnsupportedTimeSeriesError ua_plev_cube.coord = _raise_CoordinateNotFoundError with ( pytest.raises(um2nc.UnsupportedTimeSeriesError) ): - um2nc.fix_latlon_coords(ua_plev_cube, grid_type, D_LAT_N96, D_LON_N96) + um2nc.fix_latlon_coords(ua_plev_cube, um2nc.GRID_NEW_DYNAMICS, + D_LAT_N96, D_LON_N96) def test_fix_cell_methods_drop_hours(): diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 8d39d7b..a0ca014 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -297,12 +297,12 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): try: latitude_coordinate = cube.coord(LATITUDE) longitude_coordinate = cube.coord(LONGITUDE) - except iris.exceptions.CoordinateNotFoundError: + except iris.exceptions.CoordinateNotFoundError as CoordError: msg = ( "Missing latitude or longitude coordinate for variable (possible timeseries?): \n" f"{cube}\n" ) - raise UnsupportedTimeSeriesError(msg) + raise UnsupportedTimeSeriesError(msg) from CoordError # Force to double for consistency with CMOR latitude_coordinate.points = latitude_coordinate.points.astype(np.float64) From d2b4ce932ede77b823bafdedd2bd8cc6513a7a88 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 6 Sep 2024 12:36:14 +1000 Subject: [PATCH 22/27] Remove resolved TODO message --- test/test_um2netcdf.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index cf92a5f..252ecc2 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -403,11 +403,6 @@ def __init__(self, item_code, var_name=None, attributes=None, units=None): self.units = None or units self.standard_name = None self.long_name = None - # TODO: Can I remove this... It breaks DummyCubeWithCoords - # It could be required if apply_mask is called during process - # tests. Would cause a KeyError anyway? - # self.coord = {} - def coord(self): raise NotImplementedError( From 4e0d637f73dc2d17c325fe0c20a761b840fd3d79 Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 6 Sep 2024 14:56:54 +1000 Subject: [PATCH 23/27] Swap error type to new UnsupportedTimeSeries error --- test/test_conversion_driver_esm1p5.py | 30 +++++++++++++-------------- umpost/conversion_driver_esm1p5.py | 18 ++++------------ 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/test/test_conversion_driver_esm1p5.py b/test/test_conversion_driver_esm1p5.py index 3f7684d..a3cc570 100644 --- a/test/test_conversion_driver_esm1p5.py +++ b/test/test_conversion_driver_esm1p5.py @@ -3,6 +3,7 @@ import pytest from pathlib import Path import unittest.mock as mock +import umpost.um2netcdf as um2nc def test_get_esm1p5_fields_file_pattern(): @@ -98,11 +99,11 @@ def mock_process(base_mock_process): @pytest.fixture def mock_process_with_exception(mock_process): - # Add a generic exception with chosen message to mock_process. - # Yield function so that tests of different exception messages + # Add a specified exception with chosen message to mock_process. + # Yield function so that tests of different exceptions and messages # can make use of the same fixture. - def _mock_process_with_exception(error_message): - mock_process.side_effect = Exception(error_message) + def _mock_process_with_exception(error_type, error_message): + mock_process.side_effect = error_type(error_message) yield _mock_process_with_exception @@ -130,13 +131,15 @@ def test_convert_fields_file_list_success(mock_process, def test_convert_fields_file_list_fail_excepted(mock_process_with_exception): - # Hopefully this test will be unnecessary with um2nc standalone. - # Test that the "Variable can not be processed" error arising from time - # series inputs is excepted. - allowed_error_message = esm1p5_convert.ALLOWED_UM2NC_EXCEPTION_MESSAGES[ - "TIMESERIES_ERROR" - ] - mock_process_with_exception(allowed_error_message) + # Test that UnsupportedTimeSeriesErrors are excepted. + + # TODO: This test will not catch changes to the exceptions in um2nc. E.g. + # if decided to instead raise a RuntimeError in um2nc when timeseries + # are encountered, the conversion of ESM1.5 outputs would undesirably + # crash, but this test would say everything is fine, since we're + # prescribing the error that is raised. + mock_process_with_exception(um2nc.UnsupportedTimeSeriesError, + "timeseries error") fake_file_path = Path("fake_file") _, failed = esm1p5_convert.convert_fields_file_list( @@ -144,15 +147,12 @@ def test_convert_fields_file_list_fail_excepted(mock_process_with_exception): assert failed[0][0] == fake_file_path - # TODO: Testing the exception part of the reported failures will be easier - # once um2nc specific exceptions are added. - def test_convert_fields_file_list_fail_critical(mock_process_with_exception): # Test that critical exceptions which are not allowed by ALLOWED_UM2NC_EXCEPTION_MESSAGES # are raised, and hence lead to the conversion crashing. generic_error_message = "Test error" - mock_process_with_exception(generic_error_message) + mock_process_with_exception(Exception, generic_error_message) with pytest.raises(Exception) as exc_info: esm1p5_convert.convert_fields_file_list( ["fake_file"], "fake_nc_write_dir") diff --git a/umpost/conversion_driver_esm1p5.py b/umpost/conversion_driver_esm1p5.py index 9d1199a..6516547 100755 --- a/umpost/conversion_driver_esm1p5.py +++ b/umpost/conversion_driver_esm1p5.py @@ -35,13 +35,6 @@ # TODO: Confirm with Martin the below arguments are appropriate defaults. ARG_VALS = ARG_NAMES(3, 4, True, False, 0.5, False, None, None, False, False) -# TODO: um2nc standalone will raise more specific exceptions. -# See https://github.com/ACCESS-NRI/um2nc-standalone/issues/18 -# Improve exception handling here once those changes have been made. -ALLOWED_UM2NC_EXCEPTION_MESSAGES = { - "TIMESERIES_ERROR": "Variable can not be processed", -} - def get_esm1p5_fields_file_pattern(run_id: str): """ @@ -146,13 +139,10 @@ def convert_fields_file_list(fields_file_paths, nc_write_dir): um2netcdf.process(fields_file_path, nc_write_path, ARG_VALS) succeeded.append((fields_file_path, nc_write_path)) - except Exception as exc: - # TODO: Refactor once um2nc has specific exceptions - if exc.args[0] in ALLOWED_UM2NC_EXCEPTION_MESSAGES.values(): - failed.append((fields_file_path, exc)) - else: - # raise any unexpected errors - raise + except um2netcdf.UnsupportedTimeSeriesError as exc: + failed.append((fields_file_path, exc)) + + # Any unexpected errors will be raised return succeeded, failed From 7fb48cdc4e2e64d760fcf8927612c25fb617c38f Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Fri, 6 Sep 2024 16:45:42 +1000 Subject: [PATCH 24/27] Clean up based on review --- test/test_um2netcdf.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 252ecc2..537f96b 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -404,12 +404,11 @@ def __init__(self, item_code, var_name=None, attributes=None, units=None): self.standard_name = None self.long_name = None - def coord(self): + def coord(self, _): raise NotImplementedError( "coord() method not implemented on DummyCube" ) - def name(self): # mimic iris API return self.var_name @@ -664,20 +663,20 @@ def test_fix_units_do_nothing_no_um_units(ua_plev_cube): assert ua_plev_cube.units == orig # nothing should happen as there's no cube.units -def to_iris_DimCoord(points_and_name_func): +def to_iris_dimcoord(points_and_name_func): - def DimCoord_maker(): + def dimcoord_maker(): points, name = points_and_name_func() return iris.coords.DimCoord( points=points, standard_name=name ) - return DimCoord_maker + return dimcoord_maker @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lat_river_coord(): # iris DimCoord imitating UM V7.3s 1x1 degree river grid. # Must have length 180. @@ -686,7 +685,7 @@ def lat_river_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lon_river_coord(): # iris DimCoord imitating UM V7.3s 1x1 degree river grid. # Must have length 360. @@ -695,7 +694,7 @@ def lon_river_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lat_v_nd_coord(): # iris DimCoord imitating the real # lat_v grid from ESM1.5 (which uses the New Dynamics grid). @@ -707,7 +706,7 @@ def lat_v_nd_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lon_u_nd_coord(): # iris DimCoord imitating the real # lon_u grid from ESM1.5 (which uses the New Dynamics grid). @@ -718,7 +717,7 @@ def lon_u_nd_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lat_v_eg_coord(): # iris DimCoord imitating the real # lat_v grid from CM2 (which uses the End Game grid). @@ -729,7 +728,7 @@ def lat_v_eg_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lon_u_eg_coord(): # iris DimCoord imitating the real # lon_v grid from CM2 (which uses the End Game grid). @@ -740,7 +739,7 @@ def lon_u_eg_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lat_standard_nd_coord(): # iris DimCoord imitating the standard latitude # grid from ESM1.5 (which uses the New Dynamics grid). @@ -749,7 +748,7 @@ def lat_standard_nd_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lon_standard_nd_coord(): # iris DimCoord imitating the standard longitude # grid from ESM1.5 (which uses the New Dynamics grid). @@ -758,7 +757,7 @@ def lon_standard_nd_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lat_standard_eg_coord(): # iris DimCoord imitating the standard latitude # grid from CM2 (which uses the End Game grid). @@ -768,7 +767,7 @@ def lat_standard_eg_coord(): @pytest.fixture -@to_iris_DimCoord +@to_iris_dimcoord def lon_standard_eg_coord(): # iris DimCoord imitating the standard longitude # grid from CM2 (which uses the End Game grid). From 6a4c876a9de070e902ff7465b088052117b2399a Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Tue, 10 Sep 2024 14:42:55 +1000 Subject: [PATCH 25/27] Test style simplifications --- test/test_conversion_driver_esm1p5.py | 2 +- test/test_um2netcdf.py | 51 ++++++++++++++------------- umpost/um2netcdf.py | 4 +-- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/test/test_conversion_driver_esm1p5.py b/test/test_conversion_driver_esm1p5.py index a3cc570..de7f9b7 100644 --- a/test/test_conversion_driver_esm1p5.py +++ b/test/test_conversion_driver_esm1p5.py @@ -149,7 +149,7 @@ def test_convert_fields_file_list_fail_excepted(mock_process_with_exception): def test_convert_fields_file_list_fail_critical(mock_process_with_exception): - # Test that critical exceptions which are not allowed by ALLOWED_UM2NC_EXCEPTION_MESSAGES + # Test that critical unexpected exceptions # are raised, and hence lead to the conversion crashing. generic_error_message = "Test error" mock_process_with_exception(Exception, generic_error_message) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 537f96b..4ffd9f5 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -796,7 +796,11 @@ def __init__(self, dummy_cube, coords=None): self.coordinate_dict = {} def coord(self, coordinate_name): - return self.coordinate_dict[coordinate_name] + try: + return self.coordinate_dict[coordinate_name] + except KeyError: + msg = f"{self.__class__}: lacks coord for '{coordinate_name}'" + raise iris.exceptions.CoordinateNotFoundError(msg) def assert_coordinates_are_unmodified(lat_coord, lon_coord): @@ -811,14 +815,13 @@ def assert_coordinates_are_unmodified(lat_coord, lon_coord): assert coord.var_name is None -def assert_dtype_float64(lat_coord, lon_coord): - assert lat_coord.points.dtype == np.dtype("float64") - assert lon_coord.points.dtype == np.dtype("float64") +def is_float64(lat_coord, lon_coord): + return (lat_coord.points.dtype == np.dtype("float64") and + lon_coord.points.dtype == np.dtype("float64")) -def assert_has_bounds(lat_coord, lon_coord): - assert lat_coord.has_bounds() - assert lon_coord.has_bounds() +def has_bounds(lat_coord, lon_coord): + return lat_coord.has_bounds() and lon_coord.has_bounds() # Tests of fix_latlon_coords. This function converts coordinate points @@ -849,8 +852,8 @@ def test_fix_latlon_coords_river(ua_plev_cube, assert cube_lat_coord.var_name == um2nc.VAR_NAME_LAT_RIVER assert cube_lon_coord.var_name == um2nc.VAR_NAME_LON_RIVER - assert_dtype_float64(cube_lat_coord, cube_lon_coord) - assert_has_bounds(cube_lat_coord, cube_lon_coord) + assert is_float64(cube_lat_coord, cube_lon_coord) + assert has_bounds(cube_lat_coord, cube_lon_coord) def test_fix_latlon_coords_uv(ua_plev_cube, @@ -884,8 +887,8 @@ def test_fix_latlon_coords_uv(ua_plev_cube, assert lat_coordinate.var_name == um2nc.VAR_NAME_LAT_V assert lon_coordinate.var_name == um2nc.VAR_NAME_LON_U - assert_dtype_float64(lat_coordinate, lon_coordinate) - assert_has_bounds(lat_coordinate, lon_coordinate) + assert is_float64(lat_coordinate, lon_coordinate) + assert has_bounds(lat_coordinate, lon_coordinate) def test_fix_latlon_coords_standard(ua_plev_cube, @@ -927,8 +930,8 @@ def test_fix_latlon_coords_standard(ua_plev_cube, assert lat_coordinate.var_name == um2nc.VAR_NAME_LAT_STANDARD assert lon_coordinate.var_name == um2nc.VAR_NAME_LON_STANDARD - assert_dtype_float64(lat_coordinate, lon_coordinate) - assert_has_bounds(lat_coordinate, lon_coordinate) + assert is_float64(lat_coordinate, lon_coordinate) + assert has_bounds(lat_coordinate, lon_coordinate) def test_fix_latlon_coords_single_point(ua_plev_cube): @@ -951,13 +954,12 @@ def test_fix_latlon_coords_single_point(ua_plev_cube): coords=[lat_coord_single, lon_coord_single] ) - assert not lat_coord_single.has_bounds() - assert not lon_coord_single.has_bounds() + assert not has_bounds(lat_coord_single, lon_coord_single) um2nc.fix_latlon_coords(cube_with_uv_coords, um2nc.GRID_NEW_DYNAMICS, D_LAT_N96, D_LON_N96) - assert_has_bounds(lat_coord_single, lon_coord_single) + assert has_bounds(lat_coord_single, lon_coord_single) assert np.array_equal(lat_coord_single.bounds, expected_lat_bounds) assert np.array_equal(lon_coord_single.bounds, expected_lon_bounds) @@ -983,7 +985,7 @@ def test_fix_latlon_coords_has_bounds(ua_plev_cube): dummy_cube=ua_plev_cube, coords=[lat_coord, lon_coord] ) - assert_has_bounds(lat_coord, lon_coord) + assert has_bounds(lat_coord, lon_coord) um2nc.fix_latlon_coords(cube_with_uv_coords, um2nc.GRID_NEW_DYNAMICS, D_LAT_N96, D_LON_N96) @@ -997,18 +999,19 @@ def test_fix_latlon_coords_missing_coord_error(ua_plev_cube): Test that fix_latlon_coords raises the right type of error when a cube is missing coordinates. """ - def _raise_CoordinateNotFoundError(coord_name): - # Include an argument "coord_name" to mimic the signature of an - # Iris cube's ".coord" method - raise iris.exceptions.CoordinateNotFoundError(coord_name) + fake_coord = iris.coords.DimCoord( + points=np.array([1, 2, 3], dtype="float32"), + # Iris requires name to still be valid 'standard name' + standard_name="height" - # Replace coord method to raise UnsupportedTimeSeriesError - ua_plev_cube.coord = _raise_CoordinateNotFoundError + ) + + cube_with_fake_coord = DummyCubeWithCoords(ua_plev_cube, fake_coord) with ( pytest.raises(um2nc.UnsupportedTimeSeriesError) ): - um2nc.fix_latlon_coords(ua_plev_cube, um2nc.GRID_NEW_DYNAMICS, + um2nc.fix_latlon_coords(cube_with_fake_coord, um2nc.GRID_NEW_DYNAMICS, D_LAT_N96, D_LON_N96) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index a0ca014..c0c06df 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -297,12 +297,12 @@ def fix_latlon_coords(cube, grid_type, dlat, dlon): try: latitude_coordinate = cube.coord(LATITUDE) longitude_coordinate = cube.coord(LONGITUDE) - except iris.exceptions.CoordinateNotFoundError as CoordError: + except iris.exceptions.CoordinateNotFoundError as coord_error: msg = ( "Missing latitude or longitude coordinate for variable (possible timeseries?): \n" f"{cube}\n" ) - raise UnsupportedTimeSeriesError(msg) from CoordError + raise UnsupportedTimeSeriesError(msg) from coord_error # Force to double for consistency with CMOR latitude_coordinate.points = latitude_coordinate.points.astype(np.float64) From 21dc396ad4cdc792a3a7396d7a2c9129a6e8113c Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Tue, 17 Sep 2024 13:03:24 +1000 Subject: [PATCH 26/27] Clarity suggestions from review --- test/test_um2netcdf.py | 6 +++--- umpost/um2netcdf.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_um2netcdf.py b/test/test_um2netcdf.py index 4ffd9f5..a783fe7 100644 --- a/test/test_um2netcdf.py +++ b/test/test_um2netcdf.py @@ -14,8 +14,8 @@ import iris.cube import iris.coords -D_LAT_N96 = 1.25 -D_LON_N96 = 1.875 +D_LAT_N96 = 1.25 # Degrees between latitude points on N96 grid +D_LON_N96 = 1.875 # Degrees between longitude points on N96 grid @pytest.fixture @@ -800,7 +800,7 @@ def coord(self, coordinate_name): return self.coordinate_dict[coordinate_name] except KeyError: msg = f"{self.__class__}: lacks coord for '{coordinate_name}'" - raise iris.exceptions.CoordinateNotFoundError(msg) + raise CoordinateNotFoundError(msg) def assert_coordinates_are_unmodified(lat_coord, lon_coord): diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index c0c06df..16bfd31 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -225,8 +225,8 @@ def is_lat_v_grid(latitude_points, grid_type, dlat): return min_latitude == min_lat_v_eg_grid elif grid_type == GRID_NEW_DYNAMICS: return np.allclose(min_lat_v_nd_grid, min_latitude) - else: - return False # TODO: Is this situation ever valid? + + return False def is_lon_u_grid(longitude_points, grid_type, dlon): @@ -247,8 +247,8 @@ def is_lon_u_grid(longitude_points, grid_type, dlon): return min_longitude == min_lon_u_eg_grid elif grid_type == GRID_NEW_DYNAMICS: return np.allclose(min_lon_u_nd_grid, min_longitude) - else: - return False # TODO: Is this situation ever valid? + + return False def add_latlon_coord_bounds(cube_coordinate): From 37049c74acb5e36038c0e0999ffc936758d0e6ac Mon Sep 17 00:00:00 2001 From: Spencer Wong Date: Tue, 17 Sep 2024 14:35:56 +1000 Subject: [PATCH 27/27] Fix indentation --- umpost/um2netcdf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/umpost/um2netcdf.py b/umpost/um2netcdf.py index 16bfd31..a2d2f0b 100644 --- a/umpost/um2netcdf.py +++ b/umpost/um2netcdf.py @@ -263,9 +263,9 @@ def add_latlon_coord_bounds(cube_coordinate): coordinate_name = cube_coordinate.name() if coordinate_name not in [LONGITUDE, LATITUDE]: raise ValueError( - f"Wrong coordinate {coordinate_name} supplied. " - f"Expected one of {LONGITUDE}, {LATITUDE}." - ) + f"Wrong coordinate {coordinate_name} supplied. " + f"Expected one of {LONGITUDE}, {LATITUDE}." + ) # Only add bounds if not already present. if not cube_coordinate.has_bounds():