Skip to content

Commit

Permalink
Adds tests for flag_masks attribute
Browse files Browse the repository at this point in the history
(cherry picked from commit ad9b771)

Conflicts:
	compliance_checker/tests/resources.py - resolved by removing entries from STATIC_FILES that were added in unrelated commits
  • Loading branch information
benjwadams authored and mhidas committed Nov 3, 2016
1 parent a5b9aba commit c47b3b9
Show file tree
Hide file tree
Showing 3 changed files with 259 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
netcdf \20160919092000-ABOM-L3S_GHRSST-SSTfnd-AVHRR_D-1d_dn {
dimensions:
lon = 6000 ;
lat = 4500 ;
time = 1 ;
variables:
float lat(lat) ;
lat:_FillValue = 9.96921e+36f ;
lat:long_name = "latitude" ;
lat:units = "degrees_north" ;
lat:valid_min = -90.f ;
lat:valid_max = 90.f ;
lat:axis = "Y" ;
lat:comment = "Latitudes for locating data" ;
lat:standard_name = "latitude" ;
float lon(lon) ;
lon:_FillValue = 9.96921e+36f ;
lon:long_name = "longitude" ;
lon:units = "degrees_east" ;
lon:valid_min = -180.f ;
lon:valid_max = 360.f ;
lon:axis = "X" ;
lon:comment = "Longitudes for locating data" ;
lon:standard_name = "longitude" ;
int time(time) ;
time:_FillValue = -2147483647 ;
time:long_name = "reference time of sst file" ;
time:units = "seconds since 1981-01-01 00:00:00" ;
time:axis = "T" ;
time:comment = "A typical reference time for data" ;
time:standard_name = "time" ;
time:calendar = "gregorian" ;
short sea_surface_temperature(time, lat, lon) ;
sea_surface_temperature:_FillValue = -32768s ;
sea_surface_temperature:long_name = "sea surface foundation temperature" ;
sea_surface_temperature:units = "kelvin" ;
sea_surface_temperature:coordinates = "time lat lon" ;
sea_surface_temperature:comment = "An estimate of the foundation temperature of the upper few meters of the ocean in the absence of diurnal variation" ;
sea_surface_temperature:standard_name = "sea_surface_foundation_temperature" ;
sea_surface_temperature:add_offset = 288.177505493164 ;
sea_surface_temperature:scale_factor = 0.00999999977648258 ;
sea_surface_temperature:valid_min = -32767s ;
sea_surface_temperature:valid_max = 32767s ;
int sst_dtime(time, lat, lon) ;
sst_dtime:_FillValue = -2147483647 ;
sst_dtime:long_name = "time difference from reference time" ;
sst_dtime:units = "second" ;
sst_dtime:coordinates = "time lat lon" ;
sst_dtime:comment = "time plus sst_dtime gives seconds after 00:00:00 UTC January 1, 1981" ;
sst_dtime:add_offset = -1953.99356558919 ;
sst_dtime:scale_factor = 2.43249785252626e-05 ;
sst_dtime:valid_min = -2147483645 ;
sst_dtime:valid_max = 2147483645 ;
byte dt_analysis(time, lat, lon) ;
dt_analysis:_FillValue = -128b ;
dt_analysis:long_name = "deviation from last SST analysis" ;
dt_analysis:units = "kelvin" ;
dt_analysis:coordinates = "time lat lon" ;
dt_analysis:comment = "The difference between this SST and the previous day\'s SST" ;
dt_analysis:source = "ABOM-L4LRfnd-GLOB-GAMSSA_28km" ;
dt_analysis:add_offset = -3.1393518447876 ;
dt_analysis:scale_factor = 0.0612966137912434 ;
dt_analysis:valid_min = -127b ;
dt_analysis:valid_max = 127b ;
byte wind_speed(time, lat, lon) ;
wind_speed:_FillValue = -128b ;
wind_speed:long_name = "wind speed" ;
wind_speed:units = "m s-1" ;
wind_speed:coordinates = "time lat lon" ;
wind_speed:comment = "Typically represent surface winds (10 meters above the sea surface)" ;
wind_speed:standard_name = "wind_speed" ;
wind_speed:source = "ACCESSG-ABOM-Forecast-WSP" ;
wind_speed:height = "10m" ;
wind_speed:add_offset = 11.8505494594574 ;
wind_speed:scale_factor = 0.0762138837882181 ;
wind_speed:valid_min = -127b ;
wind_speed:valid_max = 127b ;
byte wind_speed_dtime_from_sst(time, lat, lon) ;
wind_speed_dtime_from_sst:_FillValue = -128b ;
wind_speed_dtime_from_sst:long_name = "time difference of wind speed measurement from sst measurement" ;
wind_speed_dtime_from_sst:units = "hour" ;
wind_speed_dtime_from_sst:coordinates = "time lat lon" ;
wind_speed_dtime_from_sst:comment = "The hours between the wind speed measurement and the SST observation" ;
wind_speed_dtime_from_sst:source = "ACCESSG-ABOM-Forecast-WSP" ;
wind_speed_dtime_from_sst:add_offset = -0.0324799753725529 ;
wind_speed_dtime_from_sst:scale_factor = 0.0211817006407519 ;
wind_speed_dtime_from_sst:valid_min = -127b ;
wind_speed_dtime_from_sst:valid_max = 127b ;
byte sea_ice_fraction(time, lat, lon) ;
sea_ice_fraction:_FillValue = -128b ;
sea_ice_fraction:long_name = "sea ice fraction" ;
sea_ice_fraction:units = "1" ;
sea_ice_fraction:coordinates = "time lat lon" ;
sea_ice_fraction:comment = "Fractional sea ice cover (Unitless). For spatial resolution refer to the source." ;
sea_ice_fraction:standard_name = "sea_ice_area_fraction" ;
sea_ice_fraction:source = "SSMI-NCEP-Analysis-ICE-5min" ;
sea_ice_fraction:add_offset = 0.5 ;
sea_ice_fraction:scale_factor = 0.00395256916996047 ;
sea_ice_fraction:valid_min = -127b ;
sea_ice_fraction:valid_max = 127b ;
byte sea_ice_fraction_dtime_from_sst(time, lat, lon) ;
sea_ice_fraction_dtime_from_sst:_FillValue = -128b ;
sea_ice_fraction_dtime_from_sst:long_name = "time difference of sea ice fraction measurement from sst measurement" ;
sea_ice_fraction_dtime_from_sst:units = "hour" ;
sea_ice_fraction_dtime_from_sst:coordinates = "time lat lon" ;
sea_ice_fraction_dtime_from_sst:comment = "The time difference in hours is estimated from the SST and sea ice data sets" ;
sea_ice_fraction_dtime_from_sst:source = "SSMI-NCEP-Analysis-ICE-5min" ;
sea_ice_fraction_dtime_from_sst:add_offset = 27.0402855500579 ;
sea_ice_fraction_dtime_from_sst:scale_factor = 0.124399257595596 ;
sea_ice_fraction_dtime_from_sst:valid_min = -127b ;
sea_ice_fraction_dtime_from_sst:valid_max = 127b ;
byte satellite_zenith_angle(time, lat, lon) ;
satellite_zenith_angle:_FillValue = -128b ;
satellite_zenith_angle:long_name = "satellite zenith angle" ;
satellite_zenith_angle:units = "angular_degree" ;
satellite_zenith_angle:coordinates = "time lat lon" ;
satellite_zenith_angle:comment = "The satellite zenith angle at the time of the SST observations" ;
satellite_zenith_angle:add_offset = 34.7895164489746 ;
satellite_zenith_angle:scale_factor = 0.2750159403081 ;
satellite_zenith_angle:valid_min = -127b ;
satellite_zenith_angle:valid_max = 127b ;
short l2p_flags(time, lat, lon) ;
l2p_flags:_FillValue = -32768s ;
l2p_flags:long_name = "L2P flags" ;
l2p_flags:valid_min = 0s ;
l2p_flags:valid_max = 32767s ;
l2p_flags:coordinates = "time lat lon" ;
l2p_flags:comment = "These flags are important to properly use the data. Data not flagged as microwave are sourced from an infrared sensor. The lake and river flags are currently not set, but defined in GDS2.0r4. The aerosol flag indicates high aerosol concentration. The analysis flag indicates high difference from analysis temperatures (differences greater than Analysis Limit). The lowwind flag indicates regions of low wind speed (typically less than the low Wind Limit) per NWP model. The highwind flag indicates regions of high wind speed (typically greater than the high Wind Limit) per NWP model. See wind limits in the comment field for the actual values. The edge flag indicates pixel sizes that are larger than Pixel Spread times the size of the pixel in the center of the field of view in either lat or lon direction. The terminator flag indicates that the sun is near the horizon. The reflector flag indicates that the satellite would receive direct reflected sunlight if the earth was a perfect mirror. The swath flag is used in gridded files to indicate if the pixel could have been seen by the satellite. delta_dn indicates that the day.night sst algorithm was different from the standard algorithm. Other flags may be populated and are for internal use and the definitions may change, so should not be relied on. Flags greater than 64 only apply to non-land pixels" ;
l2p_flags:flag_masks = 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 1024s, 2048s, 4096s, 8192s, 16384s ;
l2p_flags:flag_meanings = "microwave land ice lake river reserved aerosol analysis lowwind highwind edge terminator reflector swath delta_dn" ;
byte quality_level(time, lat, lon) ;
quality_level:_FillValue = -128b ;
quality_level:long_name = "quality level of SST pixel" ;
quality_level:valid_min = 0b ;
quality_level:valid_max = 5b ;
quality_level:coordinates = "time lat lon" ;
quality_level:comment = "These are the overall quality indicators and are used for all GHRSST SSTs. In this case they are a function of distance to cloud, satellite zenith angle, and day/night" ;
quality_level:flag_meanings = "no_data bad_data worst_quality low_quality acceptable_quality best_quality" ;
quality_level:flag_values = 0b, 1b, 2b, 3b, 4b, 5b ;
byte sses_bias(time, lat, lon) ;
sses_bias:_FillValue = -128b ;
sses_bias:long_name = "SSES bias estimate" ;
sses_bias:units = "kelvin" ;
sses_bias:coordinates = "time lat lon" ;
sses_bias:comment = "Bias estimate derived from L2P bias per https://www.ghrsst.org/ghrsst/tags-and-wgs/stval-wg/sses-description-of-schemes/" ;
sses_bias:add_offset = -0.415058895945549 ;
sses_bias:scale_factor = 0.00506022666753988 ;
sses_bias:valid_min = -127b ;
sses_bias:valid_max = 127b ;
byte sses_standard_deviation(time, lat, lon) ;
sses_standard_deviation:_FillValue = -128b ;
sses_standard_deviation:long_name = "SSES standard deviation estimate" ;
sses_standard_deviation:units = "kelvin" ;
sses_standard_deviation:coordinates = "time lat lon" ;
sses_standard_deviation:comment = "Standard deviation estimate derived from L2P standard deviation per https://www.ghrsst.org/ghrsst/tags-and-wgs/stval-wg/sses-description-of-schemes/" ;
sses_standard_deviation:add_offset = 0.531302787363529 ;
sses_standard_deviation:scale_factor = 0.00370251631783874 ;
sses_standard_deviation:valid_min = -127b ;
sses_standard_deviation:valid_max = 127b ;
byte sses_count(time, lat, lon) ;
sses_count:_FillValue = -128b ;
sses_count:long_name = "SSES count" ;
sses_count:units = "count" ;
sses_count:coordinates = "time lat lon" ;
sses_count:comment = "Weighted representative number of swath pixels, per https://www.ghrsst.org/ghrsst/tags-and-wgs/stval-wg/sses-description-of-schemes/. EXPERIMENTAL_FIELD" ;
sses_count:add_offset = 8.63768392801285 ;
sses_count:scale_factor = 0.0602486496386321 ;
sses_count:valid_min = -127b ;
sses_count:valid_max = 127b ;
byte sst_count(time, lat, lon) ;
sst_count:_FillValue = -128b ;
sst_count:long_name = "Number of SST measurements" ;
sst_count:units = "count" ;
sst_count:coordinates = "time lat lon" ;
sst_count:comment = "Unweighted count of number of contributory SST measurements, per https://www.ghrsst.org/ghrsst/tags-and-wgs/stval-wg/sses-description-of-schemes/. EXPERIMENTAL_FIELD" ;
sst_count:add_offset = 2.5 ;
sst_count:scale_factor = 0.0118577075098814 ;
sst_count:valid_min = -127b ;
sst_count:valid_max = 127b ;
short sst_mean(time, lat, lon) ;
sst_mean:_FillValue = -32768s ;
sst_mean:long_name = "Unweighted SST mean" ;
sst_mean:units = "kelvin" ;
sst_mean:coordinates = "time lat lon" ;
sst_mean:comment = "Unweighted mean of contributory SST measurements, per https://www.ghrsst.org/ghrsst/tags-and-wgs/stval-wg/sses-description-of-schemes/. EXPERIMENTAL_FIELD" ;
sst_mean:add_offset = 288.00749206543 ;
sst_mean:scale_factor = 0.00999999977648258 ;
sst_mean:valid_min = -32767s ;
sst_mean:valid_max = 32767s ;
byte sst_standard_deviation(time, lat, lon) ;
sst_standard_deviation:_FillValue = -128b ;
sst_standard_deviation:long_name = "Unweighted SST standard deviation" ;
sst_standard_deviation:units = "kelvin" ;
sst_standard_deviation:coordinates = "time lat lon" ;
sst_standard_deviation:comment = "Standard deviation estimate of contributory SST measurements, per https://www.ghrsst.org/ghrsst/tags-and-wgs/stval-wg/sses-description-of-schemes/. EXPERIMENTAL_FIELD" ;
sst_standard_deviation:add_offset = 2.51643705368042 ;
sst_standard_deviation:scale_factor = 0.0198927830330468 ;
sst_standard_deviation:valid_min = -127b ;
sst_standard_deviation:valid_max = 127b ;

// global attributes:
:title = "IMOS L3S Day and Night gridded multiple-sensor multiple-swath Australian region HRPT AVHRR foundation SST" ;
:summary = "A merged, day+night, multi-sensor L3S foundation SST product from IMOS level L3C gridded single sensor composites." ;
:references = "http://imos.org.au/srsdoc.html" ;
:institution = "ABOM" ;
:comment = "HRPT AVHRR experimental L3 retrieval produced by the Australian Bureau of Meteorology as a contribution to the Integrated Marine Observing System. SSTs were calibrated to drifting buoy depths (~20-30cm) under surface mixing wind conditions (>2m/s night, >6m/s day). SSTdepth observations were rejected if wind speeds were <2 m/s (night) or <6 m/s (day) to eliminate effects of potential diurnal warming and produce an estimate of foundation SST. SSTs are a weighted average of the SSTs of contributing pixels (weighted by sses_standard_deviation^-2).\nWARNING: some applications are unable to properly handle signed byte values. If byte values >127 are encountered, subtract 256 from this reported value. GRID:CONTINENTAL, SYSCODE:PRODUCTION" ;
:license = "GHRSST protocol describes data use as free and open" ;
:id = "AVHRR_D-ABOM-L3S-v01.0" ;
:naming_authority = "org.ghrsst" ;
:product_version = "01.0" ;
:uuid = "0639934b-9cfa-43e4-829c-b3dc6bca4f23" ;
:gds_version_id = "2.0r4" ;
:netcdf_version_id = "4.3.2" ;
:date_created = "20160926T021531Z" ;
:file_quality_level = 3 ;
:spatial_resolution = "0.02 deg" ;
:start_time = "20160918T181648Z" ;
:time_coverage_start = "20160918T181648Z" ;
:stop_time = "20160919T231803Z" ;
:time_coverage_end = "20160919T231803Z" ;
:northernmost_latitude = 19.99f ;
:southernmost_latitude = -69.99f ;
:easternmost_longitude = -170.01f ;
:westernmost_longitude = 70.01f ;
:source = "wind_source=ACCESSG-ABOM-Forecast-WSP,analysis_source=ABOM-L4LRfnd-GLOB-GAMSSA_28km,adi_source=unknown,ice_source=SSMI-NCEP-Analysis-ICE-5min,l3_source=AVHRR19_D-ABOM-L3C-v01.0;AVHRR18_D-ABOM-L3C-v01.0" ;
:platform = "NOAA-19;NOAA-18" ;
:sensor = "AVHRR" ;
:Metadata_Conventions = "Unidata Dataset Discovery v1.0" ;
:Metadata_Link = "TBA" ;
:keywords = "Oceans > Ocean Temperature > Sea Surface Temperature" ;
:keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Science Keywords" ;
:standard_name_vocabulary = "NetCDF Climate and Forecast (CF) Metadata Convention" ;
:geospatial_lat_units = "degrees_north" ;
:geospatial_lon_units = "degrees_east" ;
:geospatial_lat_resolution = 0.02f ;
:geospatial_lon_resolution = 0.02f ;
:creator_name = "Australian Bureau of Meteorology" ;
:creator_email = "[email protected]" ;
:creator_url = "http://www.imos.org.au/srs.html" ;
:project = "Group for High Resolution Sea Surface Temperature" ;
:publisher_name = "The GHRSST Project Office" ;
:publisher_email = "[email protected]" ;
:publisher_url = "http://www.ghrsst.org" ;
:processing_level = "L3S" ;
:cdm_data_type = "grid" ;
:history = "platform_counts=NOAA-19=2;NOAA-18=2,quality_counts=archive=4,platform=NOAA-19;NOAA-18,quality_source=archive,ice_source=SSMI-NCEP-Analysis-ICE-5min,adi_source=unknown,wind_source=ACCESSG-ABOM-Forecast-WSP,analysis_source=ABOM-L4LRfnd-GLOB-GAMSSA_28km,source_file=20160919152000-ABOM-L3C_GHRSST-SSTskin-AVHRR19_D-1d_night-v02.0-fv01.0.nc;20160919032000-ABOM-L3C_GHRSST-SSTskin-AVHRR19_D-1d_day-v02.0-fv01.0.nc;20160919152000-ABOM-L3C_GHRSST-SSTskin-AVHRR18_D-1d_night-v02.0-fv01.0.nc;20160919032000-ABOM-L3C_GHRSST-SSTskin-AVHRR18_D-1d_day-v02.0-fv01.0.nc,l3_file=20160919152000-ABOM-L3C_GHRSST-SSTskin-AVHRR19_D-1d_night-v02.0-fv01.0.nc;20160919032000-ABOM-L3C_GHRSST-SSTskin-AVHRR19_D-1d_day-v02.0-fv01.0.nc;20160919152000-ABOM-L3C_GHRSST-SSTskin-AVHRR18_D-1d_night-v02.0-fv01.0.nc;20160919032000-ABOM-L3C_GHRSST-SSTskin-AVHRR18_D-1d_day-v02.0-fv01.0.nc,l3_source=AVHRR19_D-ABOM-L3C-v01.0;AVHRR18_D-ABOM-L3C-v01.0,global_source=wind_source=ACCESSG-ABOM-Forecast-WSP,analysis_source=ABOM-L4LRfnd-GLOB-GAMSSA_28km,adi_source=unknown,ice_source=SSMI-NCEP-Analysis-ICE-5min,l3_source=AVHRR19_D-ABOM-L3C-v01.0;AVHRR18_D-ABOM-L3C-v01.0,landmask_file=lsmask.dist5.5.nc,landmask_reference=Naval Oceanographic Office (NAVOCEANO),landmask_URL=https://www.ghrsst.org/data/ghrsst-data-tools/navo-ghrsst-pp-land-sea-mask/,landmask_source=NAVOCEANO 1km Version 5.5,ice_reference=US National Weather Service - NCEP,ice_URL=http://polar.ncep.noaa.gov/seaice/Analyses.html,ice_file=20160918.ice_data.5min.nc,ice_jdate=2457650,merge_tool=mergeL3U,mergeL3U_version=6637,quality=archive,mergeL3U_quality=archive" ;
:Conventions = "CF-1.6" ;
:acknowledgment = "Any use of these data requires the following acknowledgment:\n \"HRPT AVHRR SSTfnd retrievals were produced by the Australian Bureau of Meteorology as a contribution to the Integrated Marine Observing System - an initiative of the Australian Government being conducted as part of the National Collaborative Research Infrastructure Strategy and the Super Science Initiative.\"\n The imagery data were acquired from NOAA spacecraft by the Bureau, Australian Institute of Marine Science, Australian Commonwealth Scientific and Industrial Research Organization, Geoscience Australia, and Western Australian Satellite Technology and Applications Consortium. " ;
}
1 change: 1 addition & 0 deletions compliance_checker/tests/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,5 @@ def generate_dataset(cdl_path, nc_path):
'trajectory-profile-incomplete' : get_filename('tests/data/trajectory-profile-incomplete.cdl'),
'2d-regular-grid' : get_filename('tests/data/2d-regular-grid.cdl'),
'3d-regular-grid' : get_filename('tests/data/3d-regular-grid.cdl'),
'ghrsst' : get_filename('tests/data/20160919092000-ABOM-L3S_GHRSST-SSTfnd-AVHRR_D-1d_dn.cdl'),
}
9 changes: 9 additions & 0 deletions compliance_checker/tests/test_cf.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,13 @@ def test_check_flags(self):
# has the proper message
self.assertEqual(sum(bool(re.match(m_str, msg)) for msg in messages), 7)

def test_check_flag_masks(self):
dataset = self.load_dataset(STATIC_FILES['ghrsst'])
results = self.cf.check_flags(dataset)
scored, out_of, messages = self.get_results(results)
self.assertEqual(scored, 32)
self.assertEqual(out_of, 32)

def test_check_bad_units(self):

dataset = self.load_dataset(STATIC_FILES['2dim'])
Expand Down Expand Up @@ -791,6 +798,8 @@ def test_units_temporal(self):
self.assertFalse(units_temporal('days since the big bang'))




def breakpoint(scope=None, global_scope=None):
import traceback
from IPython.config.loader import Config
Expand Down

0 comments on commit c47b3b9

Please sign in to comment.