From 0dd29c7a992cb204811715f54078a7e3faa6e368 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Fri, 3 Nov 2017 14:33:43 -0700 Subject: [PATCH 01/15] updated docs buldin script --- py_gnome/documentation/build_gh_pages.sh | 67 +++++++++++++++++++----- py_gnome/documentation/conf.py | 4 +- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/py_gnome/documentation/build_gh_pages.sh b/py_gnome/documentation/build_gh_pages.sh index 80c12caef..a295b389a 100755 --- a/py_gnome/documentation/build_gh_pages.sh +++ b/py_gnome/documentation/build_gh_pages.sh @@ -1,23 +1,66 @@ #!/bin/sh -# simple script to build and push to gh-pages -# designed to be run from master +# script to build and push to gh-pages -# make the docs -make html +# this version uses a separate copy of the repo in a parallel dir +# to keep the gh-pages branch nicely separate from the main branch. +# this makes it easier to have the index.html in the root dir +# -- to be nicely served by gh-pages + +# designed to be run from master -- or the latest release tag. + +# note that you should have the lastest version of gnome build and importable -- run the tests first! +# Maybe we should run the tests as part of this script?? + +# It's also a good idea to run "make html" first, to make sure the docs build cleanly + +# You also need another copy of the repo next to this one called: ../../../PyGnome.gh-pages/ + +# Ideally, we'd have a script that setup a conda environment, build gnome, built the docs, pushed them to gh +# +# even better, it would be set with a hook to happen whenever a new release is pushed to gitHub.. + + +GHPAGESDIR=../../../PyGnome.gh-pages/ +GHPAGES_URL=http://noaa-orr-erd.github.io/PyGnome/ -# make sure other copy of repo is in the right branch -pushd ../../../PyGnome.gh-pages/ +# make sure gh-pages dir is there -- exit if not +if [ ! -d $GHPAGESDIR ]; then + echo "To build the gitHub pages, you must first create a parallel repo: $GHPAGESDIR" + exit +fi + +if [ ! -d $GHPAGESDIR/.git ]; then + echo "To build the gitHub pages, you must first create a parallel repo: $GHPAGESDIR" + echo "It must be a git repo -- do a new git clone" + exit +fi + +# **Maybe this should be there? -- but I'm wary of doing it too automatically +# make sure that the main branch is pushed, so that pages are in sync with master +# git commit -a -m "updating before rendering docs" +# git push + +# make sure the gh pages repo is there and in the right branch +pushd $GHPAGESDIR git checkout gh-pages popd +# make the docs +make html # copy to other repo (on the gh-pages branch) -cp -R _build/html/ ../../../PyGnome.gh-pages/ +cp -vfR _build/html/* $GHPAGESDIR -pushd ../../../PyGnome.gh-pages/ -git add * # in case there are new files added -git commit -a -m "updating documentation" -# make sure we're in sync with other changes in repo -git pull -s ours +pushd $GHPAGESDIR +git add . # in case there are new files added +# NOTE: This does not remove any files! +git commit -a -m "updating rendered materials" +git branch -u origin/gh-pages # make sure we're tracking origin +git pull -s ours --no-edit # gotta pull before push.. yet maintain local updates git push +popd + +echo "Now verify the render on github.io at the following link:" +echo $GHPAGES_URL + diff --git a/py_gnome/documentation/conf.py b/py_gnome/documentation/conf.py index ef4dede69..ef752b6e3 100644 --- a/py_gnome/documentation/conf.py +++ b/py_gnome/documentation/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '0.5' +version = '0.6' # The full version, including alpha/beta/rc tags. -release = '0.5.0' +release = '0.6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 004c3da61e376cab17c7ce497294c4258e45465f Mon Sep 17 00:00:00 2001 From: "jay.hennen" Date: Wed, 8 Nov 2017 12:20:28 -0800 Subject: [PATCH 02/15] fixed bug with pywindmovers and real_data_start/stop --- py_gnome/gnome/movers/py_wind_movers.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/py_gnome/gnome/movers/py_wind_movers.py b/py_gnome/gnome/movers/py_wind_movers.py index 061c2355e..d0d9498e4 100644 --- a/py_gnome/gnome/movers/py_wind_movers.py +++ b/py_gnome/gnome/movers/py_wind_movers.py @@ -137,6 +137,26 @@ def from_netCDF(cls, uncertain_cross=uncertain_cross, default_num_method=default_num_method) + @property + def real_data_start(self): + return self.wind.time.min_time.replace(tzinfo=None) + + @real_data_start.setter + def real_data_start(self, value): + self._r_d_s = value + + @property + def real_data_stop(self): + return self.wind.time.max_time.replace(tzinfo=None) + + @real_data_stop.setter + def real_data_stop(self, value): + self._r_d_e = value + + @property + def is_data_on_cells(self): + return self.wind.grid.infer_location(self.current.u.data) != 'node' + def prepare_for_model_step(self, sc, time_step, model_time_datetime): """ Call base class method using super From bf1c9ac0a58da28e515577e5ce9415c4a02073da Mon Sep 17 00:00:00 2001 From: "jay.hennen" Date: Mon, 27 Nov 2017 11:23:30 -0800 Subject: [PATCH 03/15] added real data start/stop and extrapolate to movers and SA fixes on map --- py_gnome/gnome/map.py | 3 ++- py_gnome/gnome/movers/py_current_movers.py | 1 + py_gnome/gnome/movers/py_wind_movers.py | 18 ++++++++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/py_gnome/gnome/map.py b/py_gnome/gnome/map.py index 0a1740343..a4c2ca8f7 100644 --- a/py_gnome/gnome/map.py +++ b/py_gnome/gnome/map.py @@ -1097,7 +1097,8 @@ def __init__(self, filename, raster_size=4096 * 4096, **kwargs): else: map_bounds = BB.AsPoly() - elif spillable_area: + if spillable_area is None: + spillable_area = PolygonSet() spillable_area.append(map_bounds) diff --git a/py_gnome/gnome/movers/py_current_movers.py b/py_gnome/gnome/movers/py_current_movers.py index 4d9060fac..95ee9d2e9 100644 --- a/py_gnome/gnome/movers/py_current_movers.py +++ b/py_gnome/gnome/movers/py_current_movers.py @@ -53,6 +53,7 @@ class PyCurrentMover(movers.PyMover, serializable.Serializable): test_for_eq=False), serializable.Field('current', read=True, save_reference=True), + serializable.Field('extrapolate', read=True, save=True) ]) _state.add(update=['uncertain_duration', 'uncertain_time_delay'], save=['uncertain_duration', 'uncertain_time_delay']) diff --git a/py_gnome/gnome/movers/py_wind_movers.py b/py_gnome/gnome/movers/py_wind_movers.py index d0d9498e4..7c09242fa 100644 --- a/py_gnome/gnome/movers/py_wind_movers.py +++ b/py_gnome/gnome/movers/py_wind_movers.py @@ -2,7 +2,7 @@ import copy from colander import (SchemaNode, - Bool, Float, String, Sequence, + Bool, Float, String, Sequence, DateTime, drop) from gnome.basic_types import (oil_status, @@ -13,6 +13,8 @@ from gnome.environment import GridWind from gnome.persist import base_schema +from gnome.persist.validators import convertible_to_seconds +from gnome.persist.extend_colander import LocalDateTime class PyWindMoverSchema(base_schema.ObjType): @@ -23,6 +25,17 @@ class PyWindMoverSchema(base_schema.ObjType): extrapolate = SchemaNode(Bool(), missing=drop) time_offset = SchemaNode(Float(), missing=drop) wind = GridWind._schema(missing=drop) + real_data_start = SchemaNode(DateTime(), missing=drop) + real_data_stop = SchemaNode(DateTime(), missing=drop) + on = SchemaNode(Bool(), missing=drop) + active_start = SchemaNode(LocalDateTime(), missing=drop, + validator=convertible_to_seconds) + active_stop = SchemaNode(LocalDateTime(), missing=drop, + validator=convertible_to_seconds) + real_data_start = SchemaNode(LocalDateTime(), missing=drop, + validator=convertible_to_seconds) + real_data_stop = SchemaNode(LocalDateTime(), missing=drop, + validator=convertible_to_seconds) class PyWindMover(movers.PyMover, serializable.Serializable): @@ -33,7 +46,8 @@ class PyWindMover(movers.PyMover, serializable.Serializable): save=True, read=True, isdatafile=True, test_for_eq=False), serializable.Field('wind', save=True, read=True, - save_reference=True)]) + save_reference=True), + serializable.Field('extrapolate', read=True, save=True)]) _state.add(update=['uncertain_duration', 'uncertain_time_delay'], save=['uncertain_duration', 'uncertain_time_delay']) _schema = PyWindMoverSchema From 8f8a0ca62611498b01cbda2708ff034df6d690ab Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Mon, 27 Nov 2017 17:45:16 -0800 Subject: [PATCH 04/15] updated conda_requirements, a few things for scripting, and the check_libs function NOTE: tests can't load! --- conda_requirements.txt | 24 +++-- py_gnome/gnome/__init__.py | 101 ++++++++++++------ py_gnome/gnome/scripting/__init__.py | 18 +++- py_gnome/gnome/scripting/utilities.py | 8 +- py_gnome/gnome/spill/__init__.py | 24 ++--- py_gnome/gnome/spill/release.py | 4 + py_gnome/gnome/spill/spill.py | 62 ++++++++++- .../tests/unit_tests/test_spill/test_spill.py | 34 +++++- 8 files changed, 208 insertions(+), 67 deletions(-) diff --git a/conda_requirements.txt b/conda_requirements.txt index fc27578ea..a50fa9118 100644 --- a/conda_requirements.txt +++ b/conda_requirements.txt @@ -10,7 +10,7 @@ python=2.7.* setuptools>=23.0 -numpy=1.11.* +numpy=1.13.* scipy>=0.18.* gsw=3.0.3 psutil>=4.3 @@ -23,31 +23,35 @@ colander>=1.2 sqlalchemy>=0.7.6 zope.interface>=4.1 zope.sqlalchemy>=0.7.6 -gdal>=2.1.1 -netCDF4=1.2.7 +gdal=2.1.4 +netCDF4=1.3.1 awesome-slugify>=1.6 regex>=2014.12 unidecode>=0.04.19 pyshp=1.2.10 -gridded==0.0.9 +gridded=0.0.9 # NOAA maintained packages -unit_conversion=2.5.5 -cell_tree2d>=0.3.0 -py_gd>=0.1.5 # libgd should get brought in automatically +unit_conversion=2.5.* +cell_tree2d>=0.3.* +py_gd=0.1.* # libgd should get brought in automatically # required for building -cython=0.24.1 +cython>=0.24.* -# nice to have for development, not required to run +# needed to run tests pytest pytest-cov pytest-timeout testfixtures +# nice to have for development, not required to run matplotlib +ipython + +# required to build docs sphinx sphinx_rtd_theme nbsphinx -ipython + diff --git a/py_gnome/gnome/__init__.py b/py_gnome/gnome/__init__.py index d248d81c9..d02bbbf63 100644 --- a/py_gnome/gnome/__init__.py +++ b/py_gnome/gnome/__init__.py @@ -23,44 +23,81 @@ # a few imports so that the basic stuff is there - def check_dependency_versions(): - ''' + """ + much simpler method :-) + Checks the versions of the following libraries: gridded oillibrary unit_conversion - If the version is not at least as current as what's in the conda_requirements file, + If the version is not at least as current as what's defined here a warning is displayed - ''' - def get_version(package): - package = package.lower() - return next((p.version for p in pkg_resources.working_set if p.project_name.lower() == package), "No match") - libs = [('gridded', '>=', '0.0.9'), - ('oil-library', '>=', '1.0.0'), - ('unit-conversion', '>=', '2.5.5')] -# condafiledir = os.path.relpath(__file__).split(__file__.split('\\')[-3])[0] -# condafile = os.path.join(condafiledir, 'conda_requirements.txt') -# with open(condafile, 'r') as conda_reqs: -# for line in conda_reqs.readlines(): - for req in libs: - criteria = None - req_name, cmp_str, reqd_ver = req - if '>' in cmp_str: - criteria = (lambda a, b: a >= b) if '=' in cmp_str else (lambda a, b: a > b) - elif '<' in cmp_str: - criteria = (lambda a, b: a <= b) if '=' in cmp_str else (lambda a, b: a < b) - else: - criteria = (lambda a, b: a == b) - inst_ver = get_version(req_name) - module_ver = importlib.import_module(req_name.replace('-','_')).__version__ - if not criteria(inst_ver, reqd_ver): - if criteria(module_ver, reqd_ver): - w = 'Version {0} of {1} package is reported, but actual version in module is {2}'.format(inst_ver, req_name, module_ver) - warnings.warn(w) - else: - w = 'Version {0} of {1} package is installed in environment, {2}{3} required'.format(inst_ver, req_name, cmp_str, reqd_ver) - warnings.warn(w) + """ + libs = [('gridded', '0.0.9'), + ('oil_library', '1.0.6'), + ('unit_conversion', '2.5.5')] + + for name, version in libs: + # import the lib: + try: + module = importlib.import_module(name) + except ImportError: + print ("ERROR: The {} package, version >= {} needs to be installed". + format(name, version)) + sys.exit(1) + if module.__version__ < version: + w = ('Version {0} of {1} package is reported, but actual version in module is {2}'. + format(version, name, module.__version__)) + warnings.warn(w) + + +# ## maybe too complex that required... +# def check_dependency_versions(): +# ''' +# Checks the versions of the following libraries: +# gridded +# oillibrary +# unit_conversion +# If the version is not at least as current as what's in the conda_requirements file, +# a warning is displayed +# ''' +# def get_version(package): +# package = package.lower() +# return next((p.version for p in pkg_resources.working_set +# if p.project_name.lower() == package), "No match") + +# libs = [('gridded', '>=', '0.0.9'), +# ('oil-library', '>=', '1.0.6'), +# ('unit-conversion', '>=', '2.5.5')] +# # condafiledir = os.path.relpath(__file__).split(__file__.split('\\')[-3])[0] +# # condafile = os.path.join(condafiledir, 'conda_requirements.txt') +# # with open(condafile, 'r') as conda_reqs: +# # for line in conda_reqs.readlines(): +# for req in libs: +# criteria = None +# req_name, cmp_str, reqd_ver = req +# if '>' in cmp_str: +# criteria = (lambda a, b: a >= b) if '=' in cmp_str else (lambda a, b: a > b) +# elif '<' in cmp_str: +# criteria = (lambda a, b: a <= b) if '=' in cmp_str else (lambda a, b: a < b) +# else: +# criteria = (lambda a, b: a == b) +# inst_ver = get_version(req_name) + +# try: +# module_ver = importlib.import_module(req_name.replace('-', '_')).__version__ +# except ImportError: +# print "ERROR: The {} package, version {} {} needs to be installed".format(*req) +# sys.exit(1) + +# if not criteria(inst_ver, reqd_ver): +# if criteria(module_ver, reqd_ver): +# w = 'Version {0} of {1} package is reported, but actual version in module is {2}'.format(inst_ver, req_name, module_ver) +# warnings.warn(w) +# else: +# w = 'Version {0} of {1} package is installed in environment, {2}{3} required'.format(inst_ver, req_name, cmp_str, reqd_ver) +# warnings.warn(w) def initialize_log(config, logfile=None): diff --git a/py_gnome/gnome/scripting/__init__.py b/py_gnome/gnome/scripting/__init__.py index 30f7a92b7..541bc9aa5 100644 --- a/py_gnome/gnome/scripting/__init__.py +++ b/py_gnome/gnome/scripting/__init__.py @@ -2,13 +2,23 @@ Scripting package for GNOME with assorted utilities that make it easier to write scripts. +The ultimate goal is to be able to run py_gnome for the "common" use cases +with only functions available in this module + Helper functions are imported from various py_gnome modules (spill, environment, movers etc). """ from .utilities import * + from gnome.environment.wind import constant_wind -from gnome.spill.spill import point_line_release_spill -from gnome.spill.spill import surface_point_line_spill, subsurface_plume_spill -from gnome.movers.wind_movers import constant_wind_mover -from gnome.movers.wind_movers import wind_mover_from_file + +from gnome.spill.spill import (point_line_release_spill, + surface_point_line_spill, + subsurface_plume_spill, + grid_spill, + ) +from gnome.movers.wind_movers import (constant_wind_mover, + wind_mover_from_file, + ) + diff --git a/py_gnome/gnome/scripting/utilities.py b/py_gnome/gnome/scripting/utilities.py index 6a77e6b36..a2488891f 100644 --- a/py_gnome/gnome/scripting/utilities.py +++ b/py_gnome/gnome/scripting/utilities.py @@ -3,12 +3,18 @@ assorted utilities that make it easier to write scripts to automate gnome -designed to be imported into the pacakge __init__.py +designed to be imported into the package __init__.py + +remember to add anyting new you want imported to "__all__" """ import os import shutil +__all__ = ["make_images_dir", + "remove_netcdf", + ] + def make_images_dir(images_dir=None): """ diff --git a/py_gnome/gnome/spill/__init__.py b/py_gnome/gnome/spill/__init__.py index 5ef08fa68..f5a37f2c2 100644 --- a/py_gnome/gnome/spill/__init__.py +++ b/py_gnome/gnome/spill/__init__.py @@ -1,10 +1,13 @@ -from spill import (Spill, SpillSchema, +from spill import (Spill, + SpillSchema, point_line_release_spill, - continuous_release_spill) + continuous_release_spill, + grid_spill) + from release import (Release, ReleaseSchema, - PointLineReleaseSchema, PointLineRelease, + PointLineReleaseSchema, ContinuousRelease, SpatialRelease, GridRelease, @@ -12,18 +15,3 @@ InitElemsFromFile) import elements -__all__ = [Spill, - SpillSchema, - point_line_release_spill, - continuous_release_spill, - Release, - ReleaseSchema, - PointLineReleaseSchema, - PointLineRelease, - ContinuousRelease, - SpatialRelease, - GridRelease, - VerticalPlumeRelease, - InitElemsFromFile, - elements, - ] diff --git a/py_gnome/gnome/spill/release.py b/py_gnome/gnome/spill/release.py index 3ece58b2d..9339ef17f 100644 --- a/py_gnome/gnome/spill/release.py +++ b/py_gnome/gnome/spill/release.py @@ -920,6 +920,9 @@ def GridRelease(release_time, bounds, resolution): ((min_lon, min_lat), (max_lon, max_lat)) :type bounds: 2x2 numpy array or equivalent + + :param resolution: resolution of grid -- it will be a resoluiton X resolution grid + :type resolution: integer """ lon = np.linspace(bounds[0][0], bounds[1][0], resolution) lat = np.linspace(bounds[0][1], bounds[1][1], resolution) @@ -928,6 +931,7 @@ def GridRelease(release_time, bounds, resolution): return SpatialRelease(release_time, positions) + class ContinuousSpatialRelease(SpatialRelease): """ continuous release of elements from specified positions diff --git a/py_gnome/gnome/spill/spill.py b/py_gnome/gnome/spill/spill.py index e0cced162..981eba458 100644 --- a/py_gnome/gnome/spill/spill.py +++ b/py_gnome/gnome/spill/spill.py @@ -21,7 +21,7 @@ from gnome.persist.base_schema import ObjType from . import elements -from .release import PointLineRelease, ContinuousRelease +from .release import PointLineRelease, ContinuousRelease, GridRelease from .. import _valid_units @@ -928,6 +928,66 @@ def surface_point_line_spill(num_elements, units=units, name=name) +def grid_spill(bounds, + resolution, + release_time, + substance=None, + amount=None, + units=None, + windage_range=(.01, .04), + windage_persist=900, + name='Surface Grid Spill'): + ''' + Helper function returns a Grid Spill object + + :param bounds: bounding box of region you want the elements in: + ((min_lon, min_lat), + (max_lon, max_lat)) + :type bounds: 2x2 numpy array or equivalent + + :param resolution: resolution of grid -- it will be a resoluiton X resolution grid + :type resolution: integer + + :param release_time: time the LEs are released (datetime object) + :type release_time: datetime.datetime + + :param end_position=None: Optional. For moving source, the end position + If None, then release is from a point source + :type end_position: 3-tuple of floats (long, lat, z) + + :param end_release_time=None: optional -- for a time varying release, + the end release time. If None, then release is instantaneous + :type end_release_time: datetime.datetime + + :param substance=None: Type of oil spilled. + :type substance: str or OilProps + + :param float amount=None: mass or volume of oil spilled + + :param str units=None: units for amount spilled + + :param tuple windage_range=(.01, .04): Percentage range for windage. + Active only for surface particles + when a mind mover is added + + :param int windage_persist=900: Persistence for windage values in seconds. + Use -1 for inifinite, otherwise it is + randomly reset on this time scale + :param str name='Surface Point/Line Release': a name for the spill + ''' + release = GridRelease(release_time, bounds, resolution) + + element_type = elements.floating(windage_range=windage_range, + windage_persist=windage_persist, + substance=substance) + + return Spill(release, + element_type=element_type, + amount=amount, + units=units, + name=name) + + def subsurface_plume_spill(num_elements, start_position, diff --git a/py_gnome/tests/unit_tests/test_spill/test_spill.py b/py_gnome/tests/unit_tests/test_spill/test_spill.py index 0fa7016af..89466f10d 100644 --- a/py_gnome/tests/unit_tests/test_spill/test_spill.py +++ b/py_gnome/tests/unit_tests/test_spill/test_spill.py @@ -27,9 +27,12 @@ Release, point_line_release_spill, SpatialRelease, - InitElemsFromFile) + InitElemsFromFile, + grid_spill, + ) from gnome.spill.elements import (floating, ElementType) + from gnome.spill_container import SpillContainer from ..conftest import mock_sc_array_types, mock_append_data_arrays, test_oil, testdata @@ -1201,6 +1204,35 @@ def test_full_run(self): else: assert np.any(sc[key] != s.release._init_data[key]) +def test_grid_spill(): + + # fixme -- needs a real test! + + bounds = ((0.0, 0.0), + (10.0, 10.0)) + resolution = 20 + release_time = datetime(2017, 10, 20, 12) + spill = grid_spill(bounds, + resolution, + release_time + ) + # substance=None, + # amount=None, + # units=None, + # windage_range=(.01, .04), + # windage_persist=900, + # name='Surface Grid Spill' + # ) + print spill + + assert spill.num_elements == resolution * resolution + print vars(spill) + + assert False + + + + if __name__ == '__main__': # TC = Test_PointSourceSurfaceRelease() From ed3e33e2bb3d0691fc2e5b2a318d5eb51f2f5ae3 Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Tue, 28 Nov 2017 09:34:02 -0800 Subject: [PATCH 05/15] added wind to dissolution in tests; got rid of extra constant zero wind --- py_gnome/gnome/weatherers/dissolution.py | 5 ----- .../test_weatherers/test_dissolution.py | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/py_gnome/gnome/weatherers/dissolution.py b/py_gnome/gnome/weatherers/dissolution.py index 486a47da1..c8c2d5fa9 100644 --- a/py_gnome/gnome/weatherers/dissolution.py +++ b/py_gnome/gnome/weatherers/dissolution.py @@ -21,8 +21,6 @@ partition_coeff, droplet_avg_size) -from gnome.scripting import constant_wind - from .core import WeathererSchema from gnome.weatherers import Weatherer @@ -52,9 +50,6 @@ def __init__(self, waves=None, wind=None, **kwargs): self.waves = waves self.wind = wind - if self.wind is None: - self.wind = constant_wind(0,0) - super(Dissolution, self).__init__(**kwargs) self.array_types.update({'area': area, diff --git a/py_gnome/tests/unit_tests/test_weatherers/test_dissolution.py b/py_gnome/tests/unit_tests/test_weatherers/test_dissolution.py index bec88caec..377193e9e 100644 --- a/py_gnome/tests/unit_tests/test_weatherers/test_dissolution.py +++ b/py_gnome/tests/unit_tests/test_weatherers/test_dissolution.py @@ -46,7 +46,7 @@ def test_sort_order(): wind = constant_wind(15., 0) waves = Waves(wind, Water()) - diss = Dissolution(waves) + diss = Dissolution(waves, wind) disp = NaturalDispersion(waves=waves, water=waves.water) weathering_data = WeatheringData(water=waves.water) @@ -64,7 +64,7 @@ def test__deseriailize(): water = Water() waves = Waves(wind, water) - diss = Dissolution(waves) + diss = Dissolution(waves, wind) json_ = diss.serialize() pp.pprint(json_) @@ -83,7 +83,7 @@ def test__deseriailize(): def test_prepare_for_model_run(): 'test sort order for Dissolution weatherer' et = floating(substance='oil_bahia') - diss = Dissolution(waves) + diss = Dissolution(waves, wind) # we don't want to query the oil database, but get the sample oil assert et.substance.record.id is None @@ -110,7 +110,7 @@ def test_dissolution_k_ow(oil, temp, num_elems, k_ow, on): coefficient (K_ow) is getting calculated with reasonable values ''' et = floating(substance=oil) - diss = Dissolution(waves) + diss = Dissolution(waves, wind) (sc, time_step) = weathering_data_arrays(diss.array_types, water, element_type=et, @@ -152,7 +152,7 @@ def test_dissolution_droplet_size(oil, temp, num_elems, drop_size, on): et = floating(substance=oil) disp = NaturalDispersion(waves, water) - diss = Dissolution(waves) + diss = Dissolution(waves, wind) (sc, time_step) = weathering_data_arrays(diss.array_types, water, @@ -223,10 +223,11 @@ def test_dissolution_mass_balance(oil, temp, wind_speed, et = floating(substance=oil) waves = build_waves_obj(wind_speed, 'knots', 270, temp) + wind = waves.wind water = waves.water disp = NaturalDispersion(waves, water) - diss = Dissolution(waves) + diss = Dissolution(waves, wind) all_array_types = diss.array_types.union(disp.array_types) @@ -304,7 +305,7 @@ def test_full_run(sample_model_fcn2, oil, temp, expected_balance): model.environment += [Water(temp), wind, waves] model.weatherers += Evaporation() model.weatherers += NaturalDispersion() - model.weatherers += Dissolution(waves) + model.weatherers += Dissolution(waves, wind) for sc in model.spills.items(): print sc.__dict__.keys() @@ -365,7 +366,7 @@ def test_full_run_no_evap(sample_model_fcn2, oil, temp, expected_balance): model.environment += [Water(temp), low_wind, low_waves] # model.weatherers += Evaporation(Water(temp), low_wind) model.weatherers += NaturalDispersion(low_waves, Water(temp)) - model.weatherers += Dissolution(low_waves) + model.weatherers += Dissolution(low_waves, low_wind) print ('Model start time: {}, Duration: {}, Time step: {}' .format(model.start_time, model.duration, model.time_step)) @@ -417,7 +418,7 @@ def test_full_run_dissolution_not_active(sample_model_fcn): model.environment += [Water(288.7), wind, waves] model.weatherers += Evaporation() model.weatherers += NaturalDispersion() - model.weatherers += Dissolution(waves=waves, on=False) + model.weatherers += Dissolution(waves=waves, wind=wind, on=False) model.outputters += WeatheringOutput() for step in model: From 3ac77b0c14b0b49bf6d8222ca87d3e071d178471 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Tue, 28 Nov 2017 11:24:48 -0800 Subject: [PATCH 06/15] reverted cython version --- conda_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_requirements.txt b/conda_requirements.txt index a50fa9118..1ac2dfbcd 100644 --- a/conda_requirements.txt +++ b/conda_requirements.txt @@ -37,7 +37,7 @@ cell_tree2d>=0.3.* py_gd=0.1.* # libgd should get brought in automatically # required for building -cython>=0.24.* +cython=0.24.* # needed to run tests pytest From b551c65ae3bc8d87800f5528d908331617647cda Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Tue, 28 Nov 2017 11:57:10 -0800 Subject: [PATCH 07/15] moved gdal back again... --- conda_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_requirements.txt b/conda_requirements.txt index 1ac2dfbcd..596ac0349 100644 --- a/conda_requirements.txt +++ b/conda_requirements.txt @@ -23,7 +23,7 @@ colander>=1.2 sqlalchemy>=0.7.6 zope.interface>=4.1 zope.sqlalchemy>=0.7.6 -gdal=2.1.4 +gdal=2.1.3 netCDF4=1.3.1 awesome-slugify>=1.6 regex>=2014.12 From e5b978165c2e3ea731a2e59d76500a4015b9ee69 Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Thu, 30 Nov 2017 11:35:44 -0800 Subject: [PATCH 08/15] added wind State and make_default_refs --- py_gnome/gnome/weatherers/dissolution.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/py_gnome/gnome/weatherers/dissolution.py b/py_gnome/gnome/weatherers/dissolution.py index c8c2d5fa9..33a2b0157 100644 --- a/py_gnome/gnome/weatherers/dissolution.py +++ b/py_gnome/gnome/weatherers/dissolution.py @@ -39,6 +39,7 @@ def printoptions(*args, **kwargs): class Dissolution(Weatherer, Serializable): _state = copy.deepcopy(Weatherer._state) _state += [Field('waves', save=True, update=True, save_reference=True)] + _state += [Field('wind', save=True, update=True, save_reference=True)] _schema = WeathererSchema @@ -50,7 +51,12 @@ def __init__(self, waves=None, wind=None, **kwargs): self.waves = waves self.wind = wind - super(Dissolution, self).__init__(**kwargs) + if waves is not None and wind is not None: + make_default_refs = False + else: + make_default_refs = True + + super(Dissolution, self).__init__(make_default_refs=make_default_refs, **kwargs) self.array_types.update({'area': area, 'mass': mass, From ee3c07a8d0e4df8ae1b5a7142afb8b6c023e95dc Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Tue, 5 Dec 2017 10:00:51 -0800 Subject: [PATCH 09/15] updated filescanner for newer Cython --- py_gnome/gnome/utilities/file_tools/filescanner.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_gnome/gnome/utilities/file_tools/filescanner.pyx b/py_gnome/gnome/utilities/file_tools/filescanner.pyx index c507482e6..a17786804 100644 --- a/py_gnome/gnome/utilities/file_tools/filescanner.pyx +++ b/py_gnome/gnome/utilities/file_tools/filescanner.pyx @@ -111,7 +111,7 @@ def scan(infile, num_to_read=None): break if num_read > out_arr.shape[0]: # need to make the array bigger # NOTE: ndarray.resize does not work in Cython - out_arr.resize( ( out_arr.shape[0]*1.2, ), refcheck=False) + out_arr.resize((int(out_arr.shape[0]*1.2), ), refcheck=False) arr_view = out_arr #temp = np.zeros( (num_read+ out_arr.shape[0]*1.5) ) #temp[:num_read-1] = out_arr From 1f5f2cba95434b15b624cd1719b1aabf4a9475de Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Thu, 7 Dec 2017 11:16:17 -0800 Subject: [PATCH 10/15] Added year to oil name -it changed in the database --- py_gnome/scripts/script_ny_plume/script_ny_plume.py | 2 +- py_gnome/scripts/script_weatherers/script_weatherers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py_gnome/scripts/script_ny_plume/script_ny_plume.py b/py_gnome/scripts/script_ny_plume/script_ny_plume.py index 6f71efbca..19c6145d0 100644 --- a/py_gnome/scripts/script_ny_plume/script_ny_plume.py +++ b/py_gnome/scripts/script_ny_plume/script_ny_plume.py @@ -115,7 +115,7 @@ def make_model(images_dir=os.path.join(base_dir, 'images')): 7.2), release_time=start_time, element_type=plume(distribution=wd, - substance_name='ALASKA NORTH SLOPE (MIDDLE PIPELINE)') + substance_name='ALASKA NORTH SLOPE (MIDDLE PIPELINE, 1997)') ) model.spills += spill diff --git a/py_gnome/scripts/script_weatherers/script_weatherers.py b/py_gnome/scripts/script_weatherers/script_weatherers.py index 1da91f16e..332702af2 100644 --- a/py_gnome/scripts/script_weatherers/script_weatherers.py +++ b/py_gnome/scripts/script_weatherers/script_weatherers.py @@ -90,7 +90,7 @@ def make_model(images_dir=os.path.join(base_dir, 'images')): release_time=start_time, end_release_time=end_time, amount=1000, - substance='ALASKA NORTH SLOPE (MIDDLE PIPELINE)', + substance='ALASKA NORTH SLOPE (MIDDLE PIPELINE, 1997)', units='bbl') # set bullwinkle to .303 to cause mass goes to zero bug at 24 hours (when continuous release ends) From 15ba34f6ebc774535ccdb4f25257958bcc3e32cf Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Thu, 7 Dec 2017 12:04:10 -0800 Subject: [PATCH 11/15] removed a forced test failure... --- py_gnome/tests/unit_tests/test_spill/test_spill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_gnome/tests/unit_tests/test_spill/test_spill.py b/py_gnome/tests/unit_tests/test_spill/test_spill.py index 89466f10d..061a8d93b 100644 --- a/py_gnome/tests/unit_tests/test_spill/test_spill.py +++ b/py_gnome/tests/unit_tests/test_spill/test_spill.py @@ -1228,7 +1228,7 @@ def test_grid_spill(): assert spill.num_elements == resolution * resolution print vars(spill) - assert False + # assert False From ca01323b5c04f53432dd6654f6a4f89a7f88cd5d Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Thu, 7 Dec 2017 15:49:09 -0800 Subject: [PATCH 12/15] added positions array to array_types in init --- py_gnome/gnome/weatherers/dissolution.py | 2 ++ py_gnome/gnome/weatherers/emulsification.py | 3 ++- py_gnome/gnome/weatherers/natural_dispersion.py | 2 ++ py_gnome/gnome/weatherers/spreading.py | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/py_gnome/gnome/weatherers/dissolution.py b/py_gnome/gnome/weatherers/dissolution.py index 33a2b0157..5ef5e348a 100644 --- a/py_gnome/gnome/weatherers/dissolution.py +++ b/py_gnome/gnome/weatherers/dissolution.py @@ -16,6 +16,7 @@ from gnome.array_types import (area, mass, + positions, density, viscosity, partition_coeff, @@ -61,6 +62,7 @@ def __init__(self, waves=None, wind=None, **kwargs): self.array_types.update({'area': area, 'mass': mass, 'density': density, + 'positions': positions, 'viscosity': viscosity, 'partition_coeff': partition_coeff, 'droplet_avg_size': droplet_avg_size diff --git a/py_gnome/gnome/weatherers/emulsification.py b/py_gnome/gnome/weatherers/emulsification.py index 5098dd039..4ff16fcdc 100644 --- a/py_gnome/gnome/weatherers/emulsification.py +++ b/py_gnome/gnome/weatherers/emulsification.py @@ -14,6 +14,7 @@ mass, oil_density, density, + positions, bulltime, interfacial_area, oil_viscosity, @@ -51,7 +52,7 @@ def __init__(self, super(Emulsification, self).__init__(**kwargs) self.array_types.update({'age', 'bulltime', 'frac_water', - 'density', 'viscosity', + 'density', 'viscosity', 'positions', 'oil_density', 'oil_viscosity', 'mass', 'interfacial_area', 'frac_lost'}) diff --git a/py_gnome/gnome/weatherers/natural_dispersion.py b/py_gnome/gnome/weatherers/natural_dispersion.py index e880192e3..191113856 100644 --- a/py_gnome/gnome/weatherers/natural_dispersion.py +++ b/py_gnome/gnome/weatherers/natural_dispersion.py @@ -14,6 +14,7 @@ from gnome.array_types import (viscosity, mass, density, + positions, area, frac_water, droplet_avg_size) @@ -48,6 +49,7 @@ def __init__(self, self.array_types.update({'viscosity': viscosity, 'mass': mass, 'density': density, + 'positions': positions, 'area': area, 'frac_water': frac_water, 'droplet_avg_size': droplet_avg_size, diff --git a/py_gnome/gnome/weatherers/spreading.py b/py_gnome/gnome/weatherers/spreading.py index 924c1e8ab..0eef8af0a 100644 --- a/py_gnome/gnome/weatherers/spreading.py +++ b/py_gnome/gnome/weatherers/spreading.py @@ -550,7 +550,7 @@ def __init__(self, initialize wind to (0, 0) if it is None ''' super(Langmuir, self).__init__(**kwargs) - self.array_types.update(('area', 'fay_area', 'frac_coverage', 'spill_num', 'bulk_init_volume', 'density')) + self.array_types.update(('area', 'fay_area', 'frac_coverage', 'spill_num', 'bulk_init_volume', 'density', 'positions')) self.wind = wind From b082377fe8ab521a7c98d7faa0f0917becd36220 Mon Sep 17 00:00:00 2001 From: "jay.hennen" Date: Tue, 12 Dec 2017 17:39:49 -0800 Subject: [PATCH 13/15] added get_lines to Grid_U --- .../gnome/environment/gridded_objects_base.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/py_gnome/gnome/environment/gridded_objects_base.py b/py_gnome/gnome/environment/gridded_objects_base.py index 9ad665a2d..c57dc5e0c 100644 --- a/py_gnome/gnome/environment/gridded_objects_base.py +++ b/py_gnome/gnome/environment/gridded_objects_base.py @@ -142,6 +142,21 @@ def new_from_dict(cls, dict_): def get_cells(self): return self.nodes[self.faces] + def get_lines(self): + ''' + Returns a pair of 1D arrays. The first is an array of lengths, the + second is a 1D array of lines. The first array sequentially indexes the + second array. When the second array is split up using the first array + and the resulting lines are drawn, you should end up with a picture of + the grid. + ''' + open_cells = self.nodes[self.faces] + closed_cells = np.concatenate((open_cells, open_cells[:,None,0]), axis=1) + closed_cells = closed_cells.astype(np.float32, copy=False) + lengths = closed_cells.shape[1] * np.ones(closed_cells.shape[0], dtype=np.int32) + + return (lengths, closed_cells) + def get_nodes(self): return self.nodes[:] @@ -150,6 +165,13 @@ def get_centers(self): self.build_face_coordinates() return self.face_coordinates + def get_metadata(self): + json_ = {} + json_['nodes_shape'] = self.nodes.shape + json_['num_nodes'] = self.nodes.shape[0] + json_['num_cells'] = self.faces.shape[0] + return json_ + class Grid_S(gridded.grids.Grid_S, serializable.Serializable): @@ -219,6 +241,13 @@ def get_centers(self): else: return self.centers.reshape(-1,2) + def get_metadata(self): + json_ = {} + json_['nodes_shape'] = self.nodes.shape + json_['num_nodes'] = self.nodes.shape[0] * self.nodes.shape[1] + json_['num_cells'] = self._cell_trees['node'][2].shape[0] + return json_ + class Grid_R(gridded.grids.Grid_R, serializable.Serializable): @@ -245,6 +274,8 @@ def get_nodes(self): def get_centers(self): return self.centers.reshape(-1,2) + def get_cells(self): + return np.concatenate(self.node_lon, self.node_lat) class PyGrid(gridded.grids.Grid): From 071591d29be83137bcb7fa794c2fdf8efc496e4c Mon Sep 17 00:00:00 2001 From: "jay.hennen" Date: Wed, 13 Dec 2017 14:32:18 -0800 Subject: [PATCH 14/15] added R and S grid get_lines --- .../gnome/environment/gridded_objects_base.py | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/py_gnome/gnome/environment/gridded_objects_base.py b/py_gnome/gnome/environment/gridded_objects_base.py index c57dc5e0c..711204543 100644 --- a/py_gnome/gnome/environment/gridded_objects_base.py +++ b/py_gnome/gnome/environment/gridded_objects_base.py @@ -144,9 +144,9 @@ def get_cells(self): def get_lines(self): ''' - Returns a pair of 1D arrays. The first is an array of lengths, the - second is a 1D array of lines. The first array sequentially indexes the - second array. When the second array is split up using the first array + Returns an array of lengths, and a list of line arrays. + The first array sequentially indexes the second array. + When the second array is split up using the first array and the resulting lines are drawn, you should end up with a picture of the grid. ''' @@ -155,7 +155,7 @@ def get_lines(self): closed_cells = closed_cells.astype(np.float32, copy=False) lengths = closed_cells.shape[1] * np.ones(closed_cells.shape[0], dtype=np.int32) - return (lengths, closed_cells) + return (lengths, [closed_cells]) def get_nodes(self): return self.nodes[:] @@ -248,6 +248,21 @@ def get_metadata(self): json_['num_cells'] = self._cell_trees['node'][2].shape[0] return json_ + def get_lines(self): + ''' + Returns an array of lengths, and a list of line arrays. + The first array sequentially indexes the second array. + When the second array is split up using the first array + and the resulting lines are drawn, you should end up with a picture of + the grid. + ''' + hor_lines = np.dstack((self.node_lon[:], self.node_lat[:])).astype(np.float32, copy=False) + ver_lines = hor_lines.transpose((1, 0, 2)).astype(np.float32, copy=True) + hor_lens = hor_lines.shape[1] * np.ones(hor_lines.shape[0], dtype=np.int32) + ver_lens = ver_lines.shape[1] * np.ones(ver_lines.shape[0], dtype=np.int32) + lens = np.concatenate((hor_lens, ver_lens)) + return (lens, [hor_lines, ver_lines]) + class Grid_R(gridded.grids.Grid_R, serializable.Serializable): @@ -277,6 +292,15 @@ def get_centers(self): def get_cells(self): return np.concatenate(self.node_lon, self.node_lat) + def get_lines(self): + + lon_lines = np.array([[(lon, self.node_lat[0]), (lon, self.node_lat[len(self.node_lat)/2]), (lon, self.node_lat[-1])] for lon in self.node_lon], dtype=np.float32) + lat_lines = np.array([[(self.node_lon[0], lat), (self.node_lon[len(self.node_lon)/2], lat), (self.node_lon[-1], lat)] for lat in self.node_lat], dtype=np.float32) + lon_lens = lon_lines.shape[1] * np.ones(lon_lines.shape[0], dtype=np.int32) + lat_lens = lat_lines.shape[1] * np.ones(lat_lines.shape[0], dtype=np.int32) + lens = np.concatenate((lon_lens, lat_lens)) + return (lens, [lon_lines, lat_lines]) + class PyGrid(gridded.grids.Grid): @staticmethod From 0c735a55ae77590db08d85feb38facb4b3be078d Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Fri, 19 Jan 2018 17:15:30 -0800 Subject: [PATCH 15/15] fixed Polygon to pass a test. --- py_gnome/gnome/utilities/geometry/polygons.py | 4 +++- .../unit_tests/test_utilities/test_geometry/test_polygons.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/py_gnome/gnome/utilities/geometry/polygons.py b/py_gnome/gnome/utilities/geometry/polygons.py index 26ae545c7..a0bc11477 100644 --- a/py_gnome/gnome/utilities/geometry/polygons.py +++ b/py_gnome/gnome/utilities/geometry/polygons.py @@ -75,9 +75,11 @@ def __array_wrap__(self, out_arr, context=None): def __getitem__(self, index): """ - Override __getitem__ to return a simple (2, ) ndarray, rather than a + Override __getitem__ to return a ndarray, rather than a Polygon object """ + if type(index) is slice: + return Polygon(np.ndarray.__getitem__(self, index), self.metadata) return np.asarray(np.ndarray.__getitem__(self, index)) def __eq__(self, other): diff --git a/py_gnome/tests/unit_tests/test_utilities/test_geometry/test_polygons.py b/py_gnome/tests/unit_tests/test_utilities/test_geometry/test_polygons.py index 827604a03..33d67842e 100644 --- a/py_gnome/tests/unit_tests/test_utilities/test_geometry/test_polygons.py +++ b/py_gnome/tests/unit_tests/test_utilities/test_geometry/test_polygons.py @@ -60,6 +60,7 @@ def test_index2(self): def test_slice(self): P = Polygon([(1, 2), (3, 4), (5, 6)]) print P[:2] + print Polygon([(1, 2), (3, 4)])[:] assert P[:2] == Polygon([(1, 2), (3, 4)]) def test_metadata(self):