diff --git a/pycovjson/cli/convert.py b/pycovjson/cli/convert.py index bfec9c4..1c72236 100644 --- a/pycovjson/cli/convert.py +++ b/pycovjson/cli/convert.py @@ -19,6 +19,7 @@ def main(): :argument -v: Which variable to populate coverage with. :argument -s: [tile shape]: Tile shape. :argument -n: Use interactive mode. + :argument -u: MongoDB URL """ @@ -28,13 +29,14 @@ def main(): help='Name of input file', required=True) parser.add_argument('-o', '--output', dest='outputfile', help='Name and location of output file', default='coverage.covjson') - parser.add_argument( - '-t', '--tiled', action='store_true', help='Apply tiling') + parser.add_argument('-t', '--tiled', action='store_true', help='Apply tiling') parser.add_argument('-s', '--shape', nargs='+', help='Tile shape, list', type=int) parser.add_argument('-v', dest='variable', help='Variable to populate coverage with', required=True) parser.add_argument('-n', '--interactive', action='store_true', help='Enter interactive mode') + parser.add_argument('-u', '--endpoint_url', dest='endpoint_url', nargs=1, + help='MongoDB endpoint for CovJSON persistence') args = parser.parse_args() inputfile = args.inputfile outputfile = args.outputfile @@ -42,6 +44,7 @@ def main(): tiled = args.tiled tile_shape = args.shape interactive = args.interactive + endpoint_url = args.endpoint_url if interactive: axis = input('Which Axis?', Reader.get_axis(variable)) @@ -60,7 +63,7 @@ def main(): outputfile = outputfile.default Writer(outputfile, inputfile, [variable], - tiled=tiled, tile_shape=tile_shape).write() + tiled=tiled, tile_shape=tile_shape, endpoint_url=endpoint_url).write() if __name__ == '__main__': main() diff --git a/pycovjson/write.py b/pycovjson/write.py index e4c7187..09b4863 100644 --- a/pycovjson/write.py +++ b/pycovjson/write.py @@ -1,14 +1,16 @@ from pycovjson.model import Coverage, Domain, Parameter, Range, Reference, SpatialReferenceSystem2d, SpatialReferenceSystem3d, TemporalReferenceSystem, TileSet from pycovjson.read_netcdf import NetCDFReader as Reader -import time import json +from pymongo import MongoClient +from pymongo.son_manipulator import SONManipulator +import time import uuid class Writer(object): """Writer class""" - def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, tiled=False, tile_shape=[]) -> object: + def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, endpoint_url: object, tiled=False, tile_shape=[]) -> object: """ Writer class constructor @@ -17,6 +19,7 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj :parameter vars_to_write: List of variables to write :parameter tiled: Boolean value (default False) :parameter tile_shape: List containing shape of tiles + :parameter endpoint_url: MongoDB endpoint for CovJSON persistence """ self.output_name = output_name self.tile_shape = tile_shape @@ -41,18 +44,27 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj self.ref_list.append(SpatialReferenceSystem2d()) elif 't' not in self.axis_list and 'z' not in self.axis_list: self.ref_list.append(SpatialReferenceSystem2d()) + if endpoint_url is not None: + self.endpoint_url = endpoint_url + else: + self.endpoint_url = None def write(self): """ - Writes Coverage object to disk + Writes Coverage object to local disk or MongoDB """ coverage = self._construct_coverage() - if self.tiled: - self.save_covjson_tiled(coverage, self.output_name) + if self.endpoint_url is not None: + if self.tiled: + self.save_covjson_tiled(coverage, self.endpoint_url) + else: + self._save_covjson(coverage, self.endpoint_url) else: - self._save_covjson(coverage, self.output_name) - + if self.tiled: + self.save_covjson_tiled(coverage, self.output_name) + else: + self._save_covjson(coverage, self.output_name) def _construct_coverage(self): """ @@ -164,11 +176,11 @@ def _save_json(self, obj, path, **kw): stop = time.clock() print("Completed in: '%s' seconds." % (stop - start)) - def _save_covjson(self, obj, path): + def _save_covjson(self, obj, resource): """ Skip indentation of certain fields to make JSON more compact but still human readable - :param obj: - :param path: + :param obj: the CovJSON object to write + :param resource: either a local file path or a MongoDB endpoint """ @@ -179,31 +191,33 @@ def _save_covjson(self, obj, path): for covrange in obj['ranges'].values(): self.no_indent(covrange, 'axisNames', 'shape') self.compact(covrange, 'values') - self.save_json(obj, path, indent=2) + self.save_json(obj, resource, indent=2) - def save_covjson_tiled(self, obj, path): + def save_covjson_tiled(self, obj, resource): + """ + Skip indentation of certain fields to make JSON more compact but still human readable + :param obj: the CovJSON object to write + :param resource: either a local file path or a MongoDB endpoint """ - Skip indentation of certain fields to make JSON more compact but still human readable - :param obj: - :param path: - - """ for axis in obj['domain']['axes'].values(): self.compact(axis, 'values') for ref in obj['domain']['referencing']: self.no_indent(ref, 'coordinates') - self.save_json(obj, path, indent=2) + self.save_json(obj, resource, indent=2) - def save_json(self, obj, path, **kw): - with open(path, 'w') as fp: - print("Attempting to write CovJSON manifestation to '%s'" % (path)) - start = time.clock() - jsonstr = json.dumps(obj, cls=CustomEncoder, **kw) - fp.write(jsonstr) - stop = time.clock() - print("Completed in: '%s' seconds." % (stop - start)) + def save_json(self, obj, resource, **kw): + print("Attempting to write CovJSON manifestation to '%s'" % (resource)) + start = time.clock() + if resource[0].startswith('mongo'): + mongo_client = MongoDBClient(obj, resource).write() + else: + with open(resource, 'w') as fp: + jsonstr = json.dumps(obj, cls=CustomEncoder, **kw) + fp.write(jsonstr) + stop = time.clock() + print("Completed in: '%s' seconds." % (stop - start)) def save_covjson_range(self, obj, path): for covrange in obj['ranges'].values(): @@ -248,3 +262,23 @@ def encode(self, o): for k, v in self._replacement_map.items(): result = result.replace('"@@%s@@"' % (k,), v) return result + + +class MongoDBClient(object): + ''' + A client for persisting CovJSON objects into MongoDB + ''' + def __init__(self, covjson_obj, endpoint_url): + self.covjson_obj = covjson_obj + self.endpoint_url = endpoint_url + + def write(self): + """ + Make the MongoDB connection, get/create a DB and/or a Collection + and insert the CovJSON Document into MongoDB + """ + client = MongoClient(self.endpoint_url) + db = client.covjson_db + covjson_collection = db.covjsons + covjson_collection.insert_one(json.loads(json.dumps(self.covjson_obj, cls=CustomEncoder))) + diff --git a/setup.py b/setup.py index f0761b9..21874db 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ ] _description = 'Create CovJSON files from common scientific data formats' _downloadURL = 'http://pypi.python.org/pypi/pycovjson/' -_requirements = ["xarray","numpy", "pandas"] +_requirements = ["xarray","numpy", "pandas", "pymongo"] _keywords = ['dataset', 'coverage', 'covjson'] _license = 'Copyright :: University of Reading' _long_description = 'A python utility library for creating CovJSON files from common scientific data formats'