From 5c2957be102360775eae96f2ecc568fc7649c2df Mon Sep 17 00:00:00 2001
From: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Date: Tue, 11 Jul 2017 12:26:02 -0700
Subject: [PATCH 1/2] ISSUE-35 Add flag to pycovjson-convert to pipe data into
MongoDB
---
pycovjson/cli/convert.py | 8 +++-
pycovjson/write.py | 84 +++++++++++++++++++++++++++-------------
setup.py | 2 +-
3 files changed, 65 insertions(+), 29 deletions(-)
diff --git a/pycovjson/cli/convert.py b/pycovjson/cli/convert.py
index bfec9c4..cecbd2f 100644
--- a/pycovjson/cli/convert.py
+++ b/pycovjson/cli/convert.py
@@ -19,6 +19,7 @@ def main():
:argument -v: Which variable to populate coverage with.
:argument -s: [tile shape]: Tile shape.
:argument -n: Use interactive mode.
+ :argument -u: MongoDB URL
"""
@@ -35,6 +36,7 @@ def main():
parser.add_argument('-v', dest='variable',
help='Variable to populate coverage with', required=True)
parser.add_argument('-n', '--interactive', action='store_true', help='Enter interactive mode')
+ parser.add_argument('-u', '--endpoint_url', dest='endpoint_url', nargs=1, help='MongoDB endpoint for CovJSON persistence')
args = parser.parse_args()
inputfile = args.inputfile
outputfile = args.outputfile
@@ -42,6 +44,7 @@ def main():
tiled = args.tiled
tile_shape = args.shape
interactive = args.interactive
+ endpoint_url = args.endpoint_url
if interactive:
axis = input('Which Axis?', Reader.get_axis(variable))
@@ -59,8 +62,9 @@ def main():
if outputfile == None:
outputfile = outputfile.default
- Writer(outputfile, inputfile, [variable],
- tiled=tiled, tile_shape=tile_shape).write()
+ if endpoint_url is not None:
+ Writer(outputfile, inputfile, [variable],
+ tiled=tiled, tile_shape=tile_shape, endpoint_url=endpoint_url).write()
if __name__ == '__main__':
main()
diff --git a/pycovjson/write.py b/pycovjson/write.py
index e4c7187..5761924 100644
--- a/pycovjson/write.py
+++ b/pycovjson/write.py
@@ -1,14 +1,16 @@
from pycovjson.model import Coverage, Domain, Parameter, Range, Reference, SpatialReferenceSystem2d, SpatialReferenceSystem3d, TemporalReferenceSystem, TileSet
from pycovjson.read_netcdf import NetCDFReader as Reader
-import time
import json
+from pymongo import MongoClient
+from pymongo.son_manipulator import SONManipulator
+import time
import uuid
class Writer(object):
"""Writer class"""
- def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, tiled=False, tile_shape=[]) -> object:
+ def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, endpoint_url: object, tiled=False, tile_shape=[]) -> object:
"""
Writer class constructor
@@ -17,6 +19,7 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj
:parameter vars_to_write: List of variables to write
:parameter tiled: Boolean value (default False)
:parameter tile_shape: List containing shape of tiles
+ :parameter endpoint_url: MongoDB endpoint for CovJSON persistence
"""
self.output_name = output_name
self.tile_shape = tile_shape
@@ -41,18 +44,25 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj
self.ref_list.append(SpatialReferenceSystem2d())
elif 't' not in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(SpatialReferenceSystem2d())
+ if endpoint_url is not None:
+ self.endpoint_url = endpoint_url
def write(self):
"""
- Writes Coverage object to disk
+ Writes Coverage object to local disk or MongoDB
"""
coverage = self._construct_coverage()
- if self.tiled:
- self.save_covjson_tiled(coverage, self.output_name)
+ if self.endpoint_url is not None:
+ if self.tiled:
+ self.save_covjson_tiled(coverage, self.endpoint_url)
+ else:
+ self._save_covjson(coverage, self.endpoint_url)
else:
- self._save_covjson(coverage, self.output_name)
-
+ if self.tiled:
+ self.save_covjson_tiled(coverage, self.output_name)
+ else:
+ self._save_covjson(coverage, self.output_name)
def _construct_coverage(self):
"""
@@ -164,11 +174,11 @@ def _save_json(self, obj, path, **kw):
stop = time.clock()
print("Completed in: '%s' seconds." % (stop - start))
- def _save_covjson(self, obj, path):
+ def _save_covjson(self, obj, resource):
"""
Skip indentation of certain fields to make JSON more compact but still human readable
- :param obj:
- :param path:
+ :param obj: the CovJSON object to write
+ :param resource: either a local file path or a MongoDB endpoint
"""
@@ -179,31 +189,33 @@ def _save_covjson(self, obj, path):
for covrange in obj['ranges'].values():
self.no_indent(covrange, 'axisNames', 'shape')
self.compact(covrange, 'values')
- self.save_json(obj, path, indent=2)
+ self.save_json(obj, resource, indent=2)
- def save_covjson_tiled(self, obj, path):
+ def save_covjson_tiled(self, obj, resource):
+ """
+ Skip indentation of certain fields to make JSON more compact but still human readable
+ :param obj: the CovJSON object to write
+ :param resource: either a local file path or a MongoDB endpoint
"""
- Skip indentation of certain fields to make JSON more compact but still human readable
- :param obj:
- :param path:
-
- """
for axis in obj['domain']['axes'].values():
self.compact(axis, 'values')
for ref in obj['domain']['referencing']:
self.no_indent(ref, 'coordinates')
- self.save_json(obj, path, indent=2)
+ self.save_json(obj, resource, indent=2)
- def save_json(self, obj, path, **kw):
- with open(path, 'w') as fp:
- print("Attempting to write CovJSON manifestation to '%s'" % (path))
- start = time.clock()
- jsonstr = json.dumps(obj, cls=CustomEncoder, **kw)
- fp.write(jsonstr)
- stop = time.clock()
- print("Completed in: '%s' seconds." % (stop - start))
+ def save_json(self, obj, resource, **kw):
+ print("Attempting to write CovJSON manifestation to '%s'" % (resource[0]))
+ start = time.clock()
+ if resource[0].startswith('mongo'):
+ mongo_client = MongoDBClient(obj, resource).write()
+ else:
+ with open(resource, 'w') as fp:
+ jsonstr = json.dumps(obj, cls=CustomEncoder, **kw)
+ fp.write(jsonstr)
+ stop = time.clock()
+ print("Completed in: '%s' seconds." % (stop - start))
def save_covjson_range(self, obj, path):
for covrange in obj['ranges'].values():
@@ -248,3 +260,23 @@ def encode(self, o):
for k, v in self._replacement_map.items():
result = result.replace('"@@%s@@"' % (k,), v)
return result
+
+
+class MongoDBClient(object):
+ '''
+ A client for persisting CovJSON objects into MongoDB
+ '''
+ def __init__(self, covjson_obj, endpoint_url):
+ self.covjson_obj = covjson_obj
+ self.endpoint_url = endpoint_url
+
+ def write(self):
+ """
+ Make the MongoDB connection, get/create a DB and/or a Collection
+ and insert the CovJSON Document into MongoDB
+ """
+ client = MongoClient(self.endpoint_url)
+ db = client.covjson_db
+ covjson_collection = db.covjsons
+ covjson_collection.insert_one(json.loads(json.dumps(self.covjson_obj, cls=CustomEncoder)))
+
diff --git a/setup.py b/setup.py
index f0761b9..21874db 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@
]
_description = 'Create CovJSON files from common scientific data formats'
_downloadURL = 'http://pypi.python.org/pypi/pycovjson/'
-_requirements = ["xarray","numpy", "pandas"]
+_requirements = ["xarray","numpy", "pandas", "pymongo"]
_keywords = ['dataset', 'coverage', 'covjson']
_license = 'Copyright :: University of Reading'
_long_description = 'A python utility library for creating CovJSON files from common scientific data formats'
From 5f83be8e4358d69ffabe8fe21b4a15d5b4a418c3 Mon Sep 17 00:00:00 2001
From: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Date: Tue, 11 Jul 2017 14:12:20 -0700
Subject: [PATCH 2/2] ISSUE-35 Add flag to pycovjson-convert to pipe data into
MongoDB
---
pycovjson/cli/convert.py | 11 +++++------
pycovjson/write.py | 4 +++-
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/pycovjson/cli/convert.py b/pycovjson/cli/convert.py
index cecbd2f..1c72236 100644
--- a/pycovjson/cli/convert.py
+++ b/pycovjson/cli/convert.py
@@ -29,14 +29,14 @@ def main():
help='Name of input file', required=True)
parser.add_argument('-o', '--output', dest='outputfile',
help='Name and location of output file', default='coverage.covjson')
- parser.add_argument(
- '-t', '--tiled', action='store_true', help='Apply tiling')
+ parser.add_argument('-t', '--tiled', action='store_true', help='Apply tiling')
parser.add_argument('-s', '--shape', nargs='+',
help='Tile shape, list', type=int)
parser.add_argument('-v', dest='variable',
help='Variable to populate coverage with', required=True)
parser.add_argument('-n', '--interactive', action='store_true', help='Enter interactive mode')
- parser.add_argument('-u', '--endpoint_url', dest='endpoint_url', nargs=1, help='MongoDB endpoint for CovJSON persistence')
+ parser.add_argument('-u', '--endpoint_url', dest='endpoint_url', nargs=1,
+ help='MongoDB endpoint for CovJSON persistence')
args = parser.parse_args()
inputfile = args.inputfile
outputfile = args.outputfile
@@ -62,9 +62,8 @@ def main():
if outputfile == None:
outputfile = outputfile.default
- if endpoint_url is not None:
- Writer(outputfile, inputfile, [variable],
- tiled=tiled, tile_shape=tile_shape, endpoint_url=endpoint_url).write()
+ Writer(outputfile, inputfile, [variable],
+ tiled=tiled, tile_shape=tile_shape, endpoint_url=endpoint_url).write()
if __name__ == '__main__':
main()
diff --git a/pycovjson/write.py b/pycovjson/write.py
index 5761924..09b4863 100644
--- a/pycovjson/write.py
+++ b/pycovjson/write.py
@@ -46,6 +46,8 @@ def __init__(self, output_name: object, dataset_path: object, vars_to_write: obj
self.ref_list.append(SpatialReferenceSystem2d())
if endpoint_url is not None:
self.endpoint_url = endpoint_url
+ else:
+ self.endpoint_url = None
def write(self):
"""
@@ -206,7 +208,7 @@ def save_covjson_tiled(self, obj, resource):
self.save_json(obj, resource, indent=2)
def save_json(self, obj, resource, **kw):
- print("Attempting to write CovJSON manifestation to '%s'" % (resource[0]))
+ print("Attempting to write CovJSON manifestation to '%s'" % (resource))
start = time.clock()
if resource[0].startswith('mongo'):
mongo_client = MongoDBClient(obj, resource).write()