diff --git a/pycovjson/cli/convert.py b/pycovjson/cli/convert.py index a1fe8e3..2210b75 100644 --- a/pycovjson/cli/convert.py +++ b/pycovjson/cli/convert.py @@ -10,12 +10,18 @@ def main(): - parser = argparse.ArgumentParser(description='Convert Scientific Data Formats into CovJSON.') - parser.add_argument('-i', '--input', dest='inputfile', help='Name of input file', required=True) - parser.add_argument('-o', '--output', dest='outputfile', help='Name and location of output file', default='coverage.covjson') - parser.add_argument('-t', '--tiled', action='store_true', help='Apply tiling') - parser.add_argument('-s', '--shape', nargs='+', help='Tile shape, list', type=int) - parser.add_argument('-v', dest='variable', help='Variable to populate coverage with') + parser = argparse.ArgumentParser( + description='Convert Scientific Data Formats into CovJSON.') + parser.add_argument('-i', '--input', dest='inputfile', + help='Name of input file', required=True) + parser.add_argument('-o', '--output', dest='outputfile', + help='Name and location of output file', default='coverage.covjson') + parser.add_argument( + '-t', '--tiled', action='store_true', help='Apply tiling') + parser.add_argument('-s', '--shape', nargs='+', + help='Tile shape, list', type=int) + parser.add_argument('-v', dest='variable', + help='Variable to populate coverage with') args = parser.parse_args() inputfile = args.inputfile outputfile = args.outputfile @@ -23,23 +29,21 @@ def main(): tiled = args.tiled tile_shape = args.shape - if tiled and len(tile_shape) == 0: reader = Reader(inputfile) shape_list = reader.get_shape(variable) dims = reader.get_dimensions(variable) print(list(zip(dims, shape_list))) - tile_shape = input('Enter the shape tile shape as a list of comma separated integers') + tile_shape = input( + 'Enter the shape tile shape as a list of comma separated integers') tile_shape = tile_shape.split(',') tile_shape = list(map(int, tile_shape)) print(tile_shape) if outputfile == None: outputfile = outputfile.default - Writer(outputfile, inputfile, [variable], tiled=tiled, tile_shape=tile_shape).write() + Writer(outputfile, inputfile, [variable], + tiled=tiled, tile_shape=tile_shape).write() if __name__ == '__main__': main() - - - diff --git a/pycovjson/cli/viewer.py b/pycovjson/cli/viewer.py index 7e273d4..d3f8b04 100644 --- a/pycovjson/cli/viewer.py +++ b/pycovjson/cli/viewer.py @@ -12,9 +12,11 @@ def main(): parser = argparse.ArgumentParser(description='View Scientific Data files.') - parser.add_argument('-i', '--input', dest='inputfile', help='Name of input file', required=True) + parser.add_argument('-i', '--input', dest='inputfile', + help='Name of input file', required=True) - parser.add_argument('-v', '--variables,', dest='variables', help='Display variables', action='store_true') + parser.add_argument('-v', '--variables,', dest='variables', + help='Display variables', action='store_true') args = parser.parse_args() inputfile = args.inputfile @@ -30,6 +32,3 @@ def main(): if __name__ == '__main__': main() - - - diff --git a/pycovjson/main.py b/pycovjson/main.py index 312a27b..d5f244e 100644 --- a/pycovjson/main.py +++ b/pycovjson/main.py @@ -39,5 +39,4 @@ vars = ['temperature', 'salinity'] - Writer('file_name.json', coverage, vars, 'NdArray').write() diff --git a/pycovjson/model.py b/pycovjson/model.py index 39c5b85..040a734 100644 --- a/pycovjson/model.py +++ b/pycovjson/model.py @@ -1,8 +1,11 @@ -import numpy as np, math +import numpy as np +import math from collections import OrderedDict + class Coverage(object): - def __init__(self,domain, ranges, params, reference): + + def __init__(self, domain, ranges, params, reference): self.domain = domain.to_dict() self.range = ranges.to_dict() self.parameter = params.to_dict() @@ -21,6 +24,7 @@ def to_dict(self): class Domain(object): + def __init__(self, domain_type, x_values=[], y_values=[], z_values=[], t_values=[]): self.domain_type = domain_type @@ -29,19 +33,20 @@ def __init__(self, domain_type, x_values=[], y_values=[], z_values=[], t_values= self.z_values = z_values self.t_values = t_values self.referencing = [] + def __str__(self): - return 'Domain Type: ' + self.domain_type + '\nAxes:'+ str(self.axes) + return 'Domain Type: ' + self.domain_type + '\nAxes:' + str(self.axes) def to_dict(self): domain_dict = OrderedDict() domain_dict['domainType'] = self.domain_type domain_dict['axes'] = {} - domain_dict['axes']['x'] = {'values' : self.x_values} + domain_dict['axes']['x'] = {'values': self.x_values} - domain_dict['axes']['y']= {'values' : self.y_values} + domain_dict['axes']['y'] = {'values': self.y_values} - domain_dict['axes']['t'] = {'values' : self.t_values} + domain_dict['axes']['t'] = {'values': self.t_values} domain_dict['axes']['z'] = {'values': self.z_values} if len(self.z_values) == 0: # domain_dict['axes']['z']= {'values' : self.z_values} @@ -55,7 +60,8 @@ def to_dict(self): class Range(object): - def __init__(self, range_type,data_type={}, axes= [],shape=[], values=[], variable_name='', tile_sets = []): + + def __init__(self, range_type, data_type={}, axes=[], shape=[], values=[], variable_name='', tile_sets=[]): self.range_type = range_type self.data_type = data_type self.axis_names = axes @@ -64,7 +70,6 @@ def __init__(self, range_type,data_type={}, axes= [],shape=[], values=[], variab self.variable_name = variable_name self.tile_sets = tile_sets - def to_dict(self): range_dict = OrderedDict() range_dict[self.variable_name] = {} @@ -73,7 +78,7 @@ def to_dict(self): range_dict[self.variable_name]['dataType'] = self.data_type range_dict[self.variable_name]['axisNames'] = self.axis_names range_dict[self.variable_name]['shape'] = self.shape - if self.range_type == 'TiledNdArray' : + if self.range_type == 'TiledNdArray': range_dict[self.variable_name]['tileSets'] = self.tile_sets else: @@ -82,7 +87,7 @@ def to_dict(self): return range_dict - def populate(self, data_type={}, axes= [],shape=[], values=[], variable_name=''): + def populate(self, data_type={}, axes=[], shape=[], values=[], variable_name=''): """ Function to populate Range object with values @@ -95,40 +100,43 @@ def populate(self, data_type={}, axes= [],shape=[], values=[], variable_name='') class Parameter(object): - def __init__(self, variable_name='', description='' , unit='', symbol='',symbol_type='',observed_property='',op_id=None, label_langtag='en' ): + + def __init__(self, variable_name='', description='', unit='', symbol='', symbol_type='', observed_property='', op_id=None, label_langtag='en'): self.variable_name = variable_name self.param_type = 'Parameter' self.description = description - self.unit = unit + self.unit = unit self.label_langtag = label_langtag self.symbol = symbol self.symbol_type = symbol_type self.observed_property = observed_property self.op_id = op_id - def to_dict(self): param_dict = OrderedDict() param_dict[self.variable_name] = {} param_dict[self.variable_name]['type'] = self.param_type param_dict[self.variable_name]['description'] = self.description - param_dict[self.variable_name]['unit'] ={} - param_dict[self.variable_name]['unit']['label'] = {self.label_langtag : self.unit} - param_dict[self.variable_name]['symbol']={} + param_dict[self.variable_name]['unit'] = {} + param_dict[self.variable_name]['unit'][ + 'label'] = {self.label_langtag: self.unit} + param_dict[self.variable_name]['symbol'] = {} param_dict[self.variable_name]['symbol']['value'] = self.symbol param_dict[self.variable_name]['symbol']['type'] = self.symbol_type - param_dict[self.variable_name]['observedProperty']={} + param_dict[self.variable_name]['observedProperty'] = {} param_dict[self.variable_name]['observedProperty']['id'] = self.op_id - param_dict[self.variable_name]['observedProperty']['label'] = {self.label_langtag : self.observed_property} + param_dict[self.variable_name]['observedProperty'][ + 'label'] = {self.label_langtag: self.observed_property} return param_dict - class Reference(object): + def __init__(self, obj_list): self.coordinates = [] self.obj_list = obj_list print('Object list : ', self.obj_list) + def get_temporal(self, *args): return self.TemporalReferenceSystem(*args) @@ -148,6 +156,7 @@ def to_list(self): class TemporalReferenceSystem(Reference): + def __init__(self, cal=None): self.type = 'TemporalRS' self.coordinates = ['t'] @@ -156,17 +165,18 @@ def __init__(self, cal=None): self.cal = "Gregorian" else: self.cal = cal + def to_dict(self): ref_dict = OrderedDict() ref_dict['coordinates'] = self.coordinates - ref_dict['system'] ={} + ref_dict['system'] = {} ref_dict['system']['type'] = self.type ref_dict['system']['calendar'] = self.cal return ref_dict - class SpatialReferenceSystem2d(Reference): + def __init__(self): self.id = "http://www.opengis.net/def/crs/OGC/1.3/CRS84" self.type = 'GeographicCRS' @@ -181,14 +191,14 @@ def set_type(self, new_type): def to_dict(self): ref_dict = OrderedDict() ref_dict['coordinates'] = self.coordinates - ref_dict['system'] ={} + ref_dict['system'] = {} ref_dict['system']['type'] = self.type ref_dict['system']['id'] = self.id return ref_dict - class SpatialReferenceSystem3d(Reference): + def __init__(self): self.id = "http://www.opengis.net/def/crs/EPSG/0/4979" self.type = 'GeographicCRS' @@ -203,15 +213,16 @@ def set_type(self, new_type): def to_dict(self): ref_dict = OrderedDict() ref_dict['coordinates'] = self.coordinates - ref_dict['system'] ={} + ref_dict['system'] = {} ref_dict['system']['type'] = self.type ref_dict['system']['id'] = self.id return ref_dict class TileSet(object): + def __init__(self, tile_shape, url_template): - self.tile_shape = tile_shape #List containing shape + self.tile_shape = tile_shape # List containing shape self.url_template = url_template def create_tileset(self): @@ -222,7 +233,6 @@ def create_tileset(self): tileset.append(tile_dict) return tileset - def get_url_template(self, val): self.val = val @@ -235,12 +245,14 @@ def generate_url_template(self, axis_names): if len(axis_names) == 1: url_template = '{' + axis_names[0] + '}.covjson' elif len(axis_names) == 2: - url_template = '{' + axis_names[0] + '}-{' + axis_names[1] +'}.covjson' + url_template = '{' + axis_names[0] + \ + '}-{' + axis_names[1] + '}.covjson' elif len(axis_names) == 3: - url_template = '{' + axis_names[0] + '}-{' + axis_names[1] + '}-{' + axis_names[2] + '}.covjson' - + url_template = '{' + axis_names[0] + '}-{' + \ + axis_names[1] + '}-{' + axis_names[2] + '}.covjson' return url_template + def get_tiles(self, tile_shape: object, array) -> object: """ Function which yields a generator which can be leveraged to return tile arrays from an input array @@ -260,7 +272,6 @@ def step(b, dim, tile_indices): tile_count = math.ceil(self.shape[dim] / tile_shape[dim]) - for i in range(tile_count): c = b[tile_shape[dim] * i:tile_shape[dim] * (i + 1)] c = np.rollaxis(c, 1) @@ -272,14 +283,3 @@ def step(b, dim, tile_indices): def get_array_shape(self): print(self.shape) return self.shape - - - - - - - - - - - diff --git a/pycovjson/readNetCDFOOP.py b/pycovjson/readNetCDFOOP.py index 9649582..68f072e 100644 --- a/pycovjson/readNetCDFOOP.py +++ b/pycovjson/readNetCDFOOP.py @@ -2,10 +2,12 @@ from pycovjson.model import Coverage, Domain, Range, Parameter, Reference import xarray as xr from collections import OrderedDict +import datetime import pandas as pd import numpy as np import re + class NetCDFReader(object): def __init__(self, dataset_path): @@ -33,8 +35,6 @@ def get_xarray(self): self.dataset = xr.open_dataset(self.dataset_path) return self.dataset - - def close(self): self.dataset.close() @@ -44,20 +44,19 @@ def _get_domain(self): # Loads of stuff return domain + def _getRanges(self): range = Range() return range + def _getParams(self): params = Parameter() return params - def _getReference(self): reference = Reference() return reference - - def get_var_names(self, dataset): try: variable_names = [var for var in dataset.variables] @@ -65,6 +64,7 @@ def get_var_names(self, dataset): except Exception as e: print("Failed", e) return None + def get_long_names(self, dataset): long_names = [] for variable in self.get_var_names(dataset): @@ -74,18 +74,16 @@ def get_long_names(self, dataset): long_names.append(dataset[variable].standard_name) return long_names - def get_vars_with_long_name(self, dataset): try: long_names = self.get_long_names(dataset) - vars_long_names = list(zip(self.get_var_names(dataset), long_names)) + vars_long_names = list( + zip(self.get_var_names(dataset), long_names)) return vars_long_names except: pass - - - def get_shape(self,variable): + def get_shape(self, variable): """ Get shape of specifed variable, as list :param variable: String specifying variable name @@ -102,7 +100,7 @@ def get_shape(self,variable): return shape_list - def is_y(self,var): + def is_y(self, var): """ Detect whether or not specified variable is a y coord :param var: @@ -116,7 +114,7 @@ def is_y(self,var): else: return False - def is_x(self,var): + def is_x(self, var): """ Detect whether or not specified variable is an x coord :param var: @@ -136,8 +134,10 @@ def is_x(self,var): def has_time(self): time_list = ['t', 'TIME', 'time', 's', 'seconds', 'Seconds', ] for var in self.var_names: - if (self.get_units(var) in time_list): return True - if (self.get_name in time_list): return True + if (self.get_units(var) in time_list): + return True + if (self.get_name in time_list): + return True if (var in time_list): return True else: @@ -155,7 +155,7 @@ def get_time(self): return False return time_var - def get_values(self,variable): + def get_values(self, variable): """ :param variable: @@ -166,7 +166,7 @@ def get_values(self,variable): return y - def get_type(self,variable): + def get_type(self, variable): """ :param dset: NetCDF dataset object :param variable: Specified @@ -180,7 +180,7 @@ def get_type(self,variable): except Exception as e: return None - def get_dimensions(self,variable): + def get_dimensions(self, variable): """ Return dimension of specified variable :param variable: Input variable @@ -192,7 +192,7 @@ def get_dimensions(self,variable): except: return None - def get_std_name(self,variable): + def get_std_name(self, variable): """ :param variable: input variable @@ -204,7 +204,7 @@ def get_std_name(self,variable): except: return None - def get_description(self,variable): + def get_description(self, variable): """ :param variable: input variable @@ -215,7 +215,7 @@ def get_description(self,variable): except: return None - def get_name(self,variable): + def get_name(self, variable): """ :param variable: input variable @@ -226,7 +226,7 @@ def get_name(self,variable): except: return None - def get_units(self,variable): + def get_units(self, variable): """ Return :param variable: @@ -238,7 +238,7 @@ def get_units(self,variable): except: return None - def get_metadata(self,variable): + def get_metadata(self, variable): """ Returns metadata for a specified variable @@ -247,7 +247,7 @@ def get_metadata(self,variable): """ return self.dataset[variable] - def get_var_group(self,variable): + def get_var_group(self, variable): """ :param variable: @@ -255,7 +255,7 @@ def get_var_group(self,variable): """ return self.dataset[variable].group() - def get_axis(self,variable): + def get_axis(self, variable): try: axis = self.dataset[variable].axis axis = list(map(str.lower, list(axis))) @@ -268,7 +268,8 @@ def get_axis(self,variable): axes_dict = self.get_axes() for dim in self.dataset[variable].dims: - index = (list(axes_dict.keys())[list(axes_dict.values()).index(dim)]) + index = (list(axes_dict.keys())[ + list(axes_dict.values()).index(dim)]) axes_list.append(index) @@ -276,8 +277,6 @@ def get_axis(self,variable): except: print('Error in axes_dict') - - def convert_time(self, t_variable): """ Formats time objects to CovJSON compliant strings @@ -297,12 +296,10 @@ def convert_time(self, t_variable): time = pd.to_datetime(str(time)) date_list.append(time.strftime('%Y-%m-%dT%H:%M:%SZ')) - # date_list = [dates.stfrtime('%Y-%m-%dT:%H') for date in dates] return date_list def extract_var_data(self, var_names): - """ Returns dictionary containing the values in each variable specified in the variable list :type var_names: object @@ -317,7 +314,6 @@ def extract_var_data(self, var_names): print("An Error occured:", e) raise e - def get_axes(self): axes_dict = OrderedDict() @@ -327,50 +323,50 @@ def get_axes(self): z_list = ['depth', 'DEPTH'] for coord in self.dataset.coords: try: - if self.dataset[coord].axis == 'T': axes_dict['t'] = coord - if self.dataset[coord].axis == 'Z': axes_dict['z'] = coord + if self.dataset[coord].axis == 'T': + axes_dict['t'] = coord + if self.dataset[coord].axis == 'Z': + axes_dict['z'] = coord except: pass try: - if self.dataset[coord].units == 'degrees_north': axes_dict['y'] = coord - if self.dataset[coord].units == 'degrees_east': axes_dict['x'] = coord + if self.dataset[coord].units == 'degrees_north': + axes_dict['y'] = coord + if self.dataset[coord].units == 'degrees_east': + axes_dict['x'] = coord except: pass try: - if self.dataset[coord].positive in ['up', 'down']: axes_dict['z'] = coord + if self.dataset[coord].positive in ['up', 'down']: + axes_dict['z'] = coord except: pass # if coord in x_list or self.dataset[coord].standard_name in x_list: axes_dict['x'] = coord # if coord in y_list or self.dataset[coord].standard_name in y_list: axes_dict['y'] = coord - if coord in t_list or self.dataset[coord].standard_name in t_list: axes_dict['t'] = coord - if coord in z_list or self.dataset[coord].standard_name in z_list: axes_dict['z'] = coord + if coord in t_list or self.dataset[coord].standard_name in t_list: + axes_dict['t'] = coord + if coord in z_list or self.dataset[coord].standard_name in z_list: + axes_dict['z'] = coord return axes_dict - - def get_x(self): for elem in (self.dataset.coords): - if elem in ['lon', 'longitude', 'LONGITUDE','Longitude', 'x', 'X']: + if elem in ['lon', 'longitude', 'LONGITUDE', 'Longitude', 'x', 'X']: return self.dataset[elem].values try: if self.dataset[elem].axis == 'X': return self.dataset[elem].values - if self.dataset[elem].standard_name in ['lon', 'longitude', 'LONGITUDE','Longitude', 'x', 'X']: + if self.dataset[elem].standard_name in ['lon', 'longitude', 'LONGITUDE', 'Longitude', 'x', 'X']: return self.dataset[elem].values if self.dataset[elem].units == 'degrees_east': return self.dataset[elem].values except AttributeError: pass - - - - - def get_y(self): y_var = self.get_axes()['y'] @@ -381,7 +377,6 @@ def get_t(self): t_var = axis_dict['t'] return self.convert_time(t_var) - def get_z(self): for elem in (self.dataset.coords): @@ -389,14 +384,14 @@ def get_z(self): return self.dataset[elem].values try: - if self.dataset[elem].axis == 'Z': return self.dataset[elem].values + if self.dataset[elem].axis == 'Z': + return self.dataset[elem].values - if self.dataset[elem].positive in ['down', 'up']: return self.dataset[elem].values + if self.dataset[elem].positive in ['down', 'up']: + return self.dataset[elem].values except: raise AttributeError - if elem in ['z', 'Z', 'depth', 'DEPTH']: return self.dataset[elem].values - - - + if elem in ['z', 'Z', 'depth', 'DEPTH']: + return self.dataset[elem].values diff --git a/pycovjson/test/test.py b/pycovjson/test/test.py index ac2da94..368e9ce 100644 --- a/pycovjson/test/test.py +++ b/pycovjson/test/test.py @@ -1,10 +1,12 @@ from pycovjson.model import TileSet, Coverage, Reference -import os, numpy as np +import os +import numpy as np import pycovjson from pycovjson.readNetCDFOOP import NetCDFReader + def test(): dir_name = os.path.dirname(__file__) @@ -14,13 +16,10 @@ def test(): dataset_path = os.path.join(dir_name, 'testdata', testfile) dataset = np.arange(60) - dataset.reshape(10,6) + dataset.reshape(10, 6) urlTemplate = 'localhost:8080/' - - - tile_shape = [2, 1] # tileSet = TileSet(tile_shape, urlTemplate, dataset) @@ -34,6 +33,7 @@ def test(): assert len(variable_names) > 0 + def read(): dir_name = os.path.dirname(__file__) testfile = 'test_xy.nc' @@ -44,6 +44,4 @@ def read(): coverage = reader.read() - - test() diff --git a/pycovjson/write.py b/pycovjson/write.py index abffa7f..1ca8546 100644 --- a/pycovjson/write.py +++ b/pycovjson/write.py @@ -2,13 +2,15 @@ from pycovjson.model import * from pycovjson.readNetCDFOOP import NetCDFReader as Reader import numpy -import time, json, uuid +import time +import json +import uuid class Writer(object): """Writer class""" - def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, tiled=False, tile_shape=[] ) -> object: + def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, tiled=False, tile_shape=[]) -> object: """ Writer class constructor :param output_name: Name of output file @@ -25,7 +27,8 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj self.tiled = tiled if tiled: self.range_type = 'TiledNdArray' - else: self.range_type = 'NdArray' + else: + self.range_type = 'NdArray' self.dataset_path = dataset_path self.Reader = Reader(dataset_path) self.axis_dict = self.Reader.get_axes() @@ -41,7 +44,6 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj elif 't' not in self.axis_list and 'z' not in self.axis_list: self.ref_list.append(SpatialReferenceSystem2d()) - def write(self): """ Writes Coverage object to disk @@ -51,14 +53,14 @@ def write(self): self.save_covjson_tiled(coverage, self.output_name) pass - def _construct_coverage(self): """ Constructs Coverage object from constituent parts :return: coverage object """ print(self.tile_shape, type(self.tile_shape)) - coverage = Coverage(self._construct_domain(),self._construct_range(), self._construct_params(), self._construct_refs()).to_dict() + coverage = Coverage(self._construct_domain(), self._construct_range( + ), self._construct_params(), self._construct_refs()).to_dict() return coverage def _construct_domain(self): @@ -81,10 +83,8 @@ def _construct_domain(self): z_values = self.Reader.get_z().flatten().tolist() - domain = Domain(domain_type, x_values, y_values, z_values, t_values) - return domain def _construct_params(self): @@ -97,8 +97,8 @@ def _construct_params(self): unit = self.Reader.get_units(variable) symbol = self.Reader.dataset[variable].units label = self.Reader.dataset[variable].long_name - params = Parameter(description=description, variable_name=variable, symbol=symbol, unit=unit,observed_property=label) - + params = Parameter(description=description, variable_name=variable, + symbol=symbol, unit=unit, observed_property=label) return params @@ -130,14 +130,18 @@ def _construct_range(self): count = 0 for tile in tile_set_obj.get_tiles(self.tile_shape, self.Reader.dataset[variable].values): - count +=1 - range = {'ranges':Range('NdArray', data_type=variable_type, axes=tile[1], shape=variable_shape, values=tile[0].flatten().tolist()).to_dict()} - self.save_covjson_range(range, str(count) +'.json' ) - url_template = tile_set_obj.generate_url_template(axis_names=axis_names) + count += 1 + range = {'ranges': Range('NdArray', data_type=variable_type, axes=tile[ + 1], shape=variable_shape, values=tile[0].flatten().tolist()).to_dict()} + self.save_covjson_range(range, str(count) + '.json') + url_template = tile_set_obj.generate_url_template( + axis_names=axis_names) # tileset = TileSet(variable_shape, url_template).create_tileset() - tileset = [{'tileShape' : [None, 173, 301], 'urlTemplate': 'http://localhost:8080/{t}.covjson'}] + tileset = [{'tileShape': [None, 173, 301], + 'urlTemplate': 'http://localhost:8080/{t}.covjson'}] - range = Range('TiledNdArray', data_type=variable_type, axes=axis_names, tile_sets=tileset, shape=variable_shape) + range = Range('TiledNdArray', data_type=variable_type, + axes=axis_names, tile_sets=tileset, shape=variable_shape) return range else: @@ -145,12 +149,14 @@ def _construct_range(self): values = self.Reader.get_values(variable).flatten().tolist() data_type = self.Reader.get_type(variable) axes = self.Reader.get_axis(variable) - range = Range(range_type='NdArray', data_type=data_type, values=values, shape=shape,\ + range = Range(range_type='NdArray', data_type=data_type, values=values, shape=shape, variable_name=variable, axes=axis_names) return range - # Adapted from https://github.com/the-iea/ecem/blob/master/preprocess/ecem/util.py - letmaik + # Adapted from + # https://github.com/the-iea/ecem/blob/master/preprocess/ecem/util.py - + # letmaik def _save_json(self, obj, path, **kw): """Save json object to disk""" with open(path, 'w') as fp: @@ -177,6 +183,7 @@ def _save_covjson(self, obj, path): self.no_indent(range, 'axisNames', 'shape') self.compact(range, 'values') self.save_json(obj, path, indent=2) + def save_covjson_tiled(self, obj, path): """ Skip indentation of certain fields to make JSON more compact but still human readable @@ -216,10 +223,9 @@ def no_indent(self, obj, *names): obj[name] = Custom(obj[name]) - - # From http://stackoverflow.com/a/25935321 class Custom(object): + def __init__(self, value, **custom_args): self.value = value self.custom_args = custom_args @@ -227,6 +233,7 @@ def __init__(self, value, **custom_args): class CustomEncoder(json.JSONEncoder): """Custom Json Encoder class - Allows Json to be saved using custom format (no_indent, compact)""" + def __init__(self, *args, **kwargs): super(CustomEncoder, self).__init__(*args, **kwargs) self._replacement_map = {} @@ -244,5 +251,3 @@ def encode(self, o): for k, v in self._replacement_map.items(): result = result.replace('"@@%s@@"' % (k,), v) return result - - diff --git a/pycovjson/writeJSON.py b/pycovjson/writeJSON.py index dc753c8..d230c11 100644 --- a/pycovjson/writeJSON.py +++ b/pycovjson/writeJSON.py @@ -31,21 +31,21 @@ # TEST - REPLACE WITH FUNCTION TO TAKE OPTIONS FROM USER dim_list = rnc.group_vars(var_names) -num_list =list(range(0,len(dim_list))) +num_list = list(range(0, len(dim_list))) -variable_dimensions = dict(zip(num_list,dim_list)) +variable_dimensions = dict(zip(num_list, dim_list)) def get_variables(): var_names = rnc.get_var_names(dset) - num_list = list(range(0,len(var_names))) + num_list = list(range(0, len(var_names))) var_dict = dict(zip(num_list, var_names)) print(var_dict) print(variable_dimensions) choice = int(input("Enter number of variable: ")) return var_dict[choice] -user_opts =[] +user_opts = [] choice = get_variables() user_opts.append(choice) @@ -82,8 +82,6 @@ def construct_parameters(var_name): return parameters - - def construct_referencing(var_name): """ @@ -113,21 +111,19 @@ def construct_referencing(var_name): print(ref_list) print('Construct Referencing ran successfuly...') - # referencing_list.append(referencing[0]) # referencing_list.append(referencing[1]) return ref_list + def construct_tiles(var_name): - tileObj = TileSet([None, 'x', 'y'],'https://www.googledrive.com/host//') + tileObj = TileSet([None, 'x', 'y'], 'https://www.googledrive.com/host//') tileObj.create_tileset('TEST') return tileObj - -def construct_covjson(json_template, data, variables,domain_type="Grid", time=False, tiled=False): - +def construct_covjson(json_template, data, variables, domain_type="Grid", time=False, tiled=False): """ Create covJSON object, fill with data values :rtype: object @@ -145,13 +141,14 @@ def construct_covjson(json_template, data, variables,domain_type="Grid", time=F # Coordinate data x - longitude y- latitude json_template['domain']['axes']['x']['values'] = (data[longitude].tolist()) json_template['domain']['axes']['y']['values'] = data[latitude].tolist() - #Melodies landcover dataset, latitude values flipped, use flipup to correct + # Melodies landcover dataset, latitude values flipped, use flipup to correct #json_template['domain']['axes']['y']['values'] = np.flipud(data[latitude]).tolist() # json_template['domain']['referencing'] = construct_referencing(user_opts[0]) if time: json_template['domain']['axes']['t'] = {'values': []} - json_template['domain']['axes']['t']['values']= rnc.convert_time(rnc.get_time()) + json_template['domain']['axes']['t'][ + 'values'] = rnc.convert_time(rnc.get_time()) json_template['domain']['referencing'] = construct_referencing(choice) for var in variables: @@ -165,9 +162,10 @@ def construct_covjson(json_template, data, variables,domain_type="Grid", time=F json_template['ranges'][var]['tileSets'] = {} for tile in range(len(tiles.get_array_shape())): - json_template['ranges'][var]['tileSets']['tileShape'] = tile_shape - json_template['ranges'][var]['tileSets']['urlTemplate'] = tiles.get_url_template(tile) - + json_template['ranges'][var][ + 'tileSets']['tileShape'] = tile_shape + json_template['ranges'][var]['tileSets'][ + 'urlTemplate'] = tiles.get_url_template(tile) json_tile = {} tile_list = list(tiles.get_tiles(tile_shape, variable)) @@ -176,31 +174,33 @@ def construct_covjson(json_template, data, variables,domain_type="Grid", time=F json_tile['values'] = tile save_json(json_tile, 'json_tile.json') - - - else: json_template['ranges'] json_template['ranges'][var] = construct_range(var, tiled=False) - json_template['ranges'][var]["values"] ={} - json_template['ranges'][var]['values'] =(data[var].ravel().tolist()) - + json_template['ranges'][var]["values"] = {} + json_template['ranges'][var]['values'] = ( + data[var].ravel().tolist()) json_template['parameters'][var] = construct_parameters(var) - json_template['parameters'][var]['description'] = rnc.get_description(var) - json_template['parameters'][var]['unit']['label']['en'] = rnc.get_units(var) + json_template['parameters'][var][ + 'description'] = rnc.get_description(var) + json_template['parameters'][var]['unit'][ + 'label']['en'] = rnc.get_units(var) if rnc.get_std_name(var) != None: - json_template['parameters'][var]['observedProperty']['id'] = 'http://vocab.nerc.ac.uk/standard_name/' + str(rnc.get_std_name(var)) + json_template['parameters'][var]['observedProperty'][ + 'id'] = 'http://vocab.nerc.ac.uk/standard_name/' + str(rnc.get_std_name(var)) else: json_template['parameters'][var]['observedProperty']['id'] = None - json_template['parameters'][var]['observedProperty']['label']['en'] = var - + json_template['parameters'][var][ + 'observedProperty']['label']['en'] = var return json_template -# Adapted from https://github.com/the-iea/ecem/blob/master/preprocess/ecem/util.py - letmaik +# Adapted from +# https://github.com/the-iea/ecem/blob/master/preprocess/ecem/util.py - +# letmaik def save_json(obj, path, **kw): @@ -214,7 +214,8 @@ def save_json(obj, path, **kw): def save_covjson(obj, path): - # skip indentation of certain fields to make it more compact but still human readable + # skip indentation of certain fields to make it more compact but still + # human readable for axis in obj['domain']['axes'].values(): compact(axis, 'values') for ref in obj['domain']['referencing']: @@ -237,12 +238,14 @@ def no_indent(obj, *names): # From http://stackoverflow.com/a/25935321 class Custom(object): + def __init__(self, value, **custom_args): self.value = value self.custom_args = custom_args class CustomEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): super(CustomEncoder, self).__init__(*args, **kwargs) self._replacement_map = {} @@ -262,28 +265,25 @@ def encode(self, o): return result - - - def set_coords(coord_list, t): t = t x = coord_list[0] y = coord_list[1] - z= coord_list[2] - return tuple([x,y,z,t]) + z = coord_list[2] + return tuple([x, y, z, t]) + def detect_coords(dset): """ :return: Tuple containing lat and lon var names """ for var in dset.variables: - if rnc.is_x(var): x = var - if rnc.is_y(var): y = var - - - - return tuple([x,y]) + if rnc.is_x(var): + x = var + if rnc.is_y(var): + y = var + return tuple([x, y]) out_file = open(json_file, "w") @@ -291,16 +291,12 @@ def detect_coords(dset): # Test parameters -print("Coords are: " , detect_coords(dset)) +print("Coords are: ", detect_coords(dset)) print(dset) var_names = rnc.get_var_names(dset) data = rnc.extract_var_data(var_names) -json_obj = construct_covjson(json_template, rnc.extract_var_data(var_names),user_opts,domain_type="Grid" ,time=rnc.has_time(),tiled=True) +json_obj = construct_covjson(json_template, rnc.extract_var_data( + var_names), user_opts, domain_type="Grid", time=rnc.has_time(), tiled=True) save_covjson(json_obj, json_file) - - - - - diff --git a/pycovjson/writeNetCDF.py b/pycovjson/writeNetCDF.py index 880ebb4..ad0a522 100644 --- a/pycovjson/writeNetCDF.py +++ b/pycovjson/writeNetCDF.py @@ -3,19 +3,21 @@ from numpy import arange, dtype -nx = 4; ny = 4; nz=4 -ncfile = Dataset('test_xy.nc','w') +nx = 4 +ny = 4 +nz = 4 +ncfile = Dataset('test_xy.nc', 'w') # create the output data. -data_out = arange(nx*ny) +data_out = arange(nx * ny) print(data_out) -data_out.shape = (nx,ny) # reshape to 3d array +data_out.shape = (nx, ny) # reshape to 3d array # create the x and y dimensions. -ncfile.createDimension('x',nx) -ncfile.createDimension('y',ny) +ncfile.createDimension('x', nx) +ncfile.createDimension('y', ny) # ncfile.createDimension('z', nz) -data = ncfile.createVariable('data',dtype('float32').char,('x','y')) +data = ncfile.createVariable('data', dtype('float32').char, ('x', 'y')) data[:] = data_out # close the file. print(ncfile.variables) -print("Wrote file!") \ No newline at end of file +print("Wrote file!") diff --git a/setup.py b/setup.py index 5e94814..90c8f5c 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,9 @@ # Package data # ------------ -_author = 'rileywilliams' -_authorEmail = 'resc@reading.ac.uk' -_classifiers = [ +_author = 'rileywilliams' +_authorEmail = 'resc@reading.ac.uk' +_classifiers = [ 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', @@ -35,6 +35,7 @@ # Setup Metadata # -------------- + def _read(*rnames): return open(os.path.join(os.path.dirname(__file__), *rnames)).read()