diff --git a/.github/workflows/conda_test.yml b/.github/workflows/conda_test.yml new file mode 100644 index 000000000..4b29ef4ce --- /dev/null +++ b/.github/workflows/conda_test.yml @@ -0,0 +1,83 @@ +name: Run tests with miniconda +on: [push] + +jobs: + test: + name: Miniconda ${{ matrix.os }} Py${{ matrix.pyver }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + #os: ["ubuntu-latest", "windows-latest","macos-latest"] + os: ["windows-latest","macos-latest"] + pyver: ["3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: test + # environment-file: environment.yml + python-version: ${{ matrix.pyver }} + auto-activate-base: false + - shell: bash -l {0} + run: | + conda config --add channels conda-forge + - name: Setup environment + shell: bash -l {0} + run: | + git clone --depth 1 -b develop https://github.com/NOAA-ORR-ERD/adios_oil_database.git oil_database + conda install -y mamba + cd ./py_gnome + echo "Setting up conda environment" + mamba install -y python=${{ matrix.pyver }} \ + --file conda_requirements.txt \ + --file conda_requirements_build.txt \ + --file conda_requirements_test.txt \ + --file ../oil_database/adios_db/conda_requirements.txt + - name: Build Our Packages + shell: bash -l {0} + run: | + pwd + ls -la + pip install ./oil_database/adios_db/ + cd ./py_gnome + python -m pip install ./ + - name: Run pytest + shell: bash -l {0} + run: | + cd ./py_gnome/tests/unit_tests + pytest --runslow + - name: Build wheel file + shell: bash -l {0} + run: | + cd ./py_gnome + python -m pip wheel ./ + - name: Copy files to artifact folder + shell: bash -l {0} + run: | + mkdir installable_wheel_${{ matrix.os }}_Py${{ matrix.pyver }} + cp ./py_gnome/*.whl installable_wheel_${{ matrix.os }}_Py${{ matrix.pyver }} + cp ./py_gnome/conda_requirements.txt installable_wheel_${{ matrix.os }}_Py${{ matrix.pyver }} + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: installable-wheel-${{ matrix.os }}-Py${{ matrix.pyver }} + path: installable_wheel_${{ matrix.os }}_Py${{ matrix.pyver }} + + lint: + name: Flake8 linting + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: lint + python-version: 3.9 + auto-activate-base: false + - shell: bash -l {0} + run: | + conda config --add channels conda-forge + - name: Lint + shell: bash -l {0} + run: | + conda install flake8 + python -m flake8 --exit-zero --statistics py_gnome/gnome/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13105e635..3afa1b0d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,8 +5,8 @@ variables: stages: - compile - - test - build + - test cache: paths: @@ -44,22 +44,27 @@ compile_pygnome: # get the adios_db code: needed for tests and requirements - git clone --depth 1 -b develop https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.orr.noaa.gov/gnome/oil_database/oil_database.git + - cd ./py_gnome - conda install -y python=$PYTHON_VER --file conda_requirements.txt --file conda_requirements_build.txt - --file oil_database/adios_db/conda_requirements.txt + --file ../oil_database/adios_db/conda_requirements.txt # install adios_db - - pip install oil_database/adios_db/ - - - cd ./py_gnome - - python setup.py install - - - cd ../ # make sure that it won't find the source gnome package. - - python -c "import gnome" # make sure that it imports + - pip install ../oil_database/adios_db/ + + # Install pygnome + # If you want verbose output including the compiler/linker commands, + # run the following command: + # python -m build --no-isolation --config-setting cmake.verbose=true + # + # And there may be times we want to build a wheel file (.whl) + # The following command will build a wheel in the current directory: + # python -m pip wheel ./ + - python -m pip install ./ # need extra requirements for the docs - conda install -y --file conda_requirements_docs.txt - - cd py_gnome/documentation && make html # build the docs + - cd documentation && make html # build the docs artifacts: when: always expire_in: 15 days @@ -76,8 +81,10 @@ test_pygnome: allow_failure: false image: registry.orr.noaa.gov/erd/centos-conda/$IMAGE_NAME$PYTHON_VER script: - - conda install -y --file conda_requirements_test.txt + - conda install -y --file py_gnome/conda_requirements_test.txt - cd ./py_gnome/tests/unit_tests && pytest --runslow + + # save the package list if tests pass - conda list > conda_packages.txt # Ideally, all the scripts in "testing_scripts" should run without errors, @@ -109,56 +116,78 @@ test_pygnome_windows: - windows - visual-studio-22 allow_failure: true + cache: [] before_script: + - echo $CI_PIPELINE_ID + + # where are we? - Get-Location - - dir + - Get-ChildItem | % { $_.FullName } # this should list the current folder + - $Env:PathOrig = $Env:Path + - $Env:CurrDir = Get-Location + + # Where is our conda installation? + # This will tell us where python is installed. This could change periodically + #- Get-ChildItem -Path 'C:\Python310' -Recurse -Include "python.exe", "conda.exe" | % { $_.FullName } + #- Get-ChildItem -Path 'C:\ProgramData\miniforge3' -Recurse -Include "python.exe", "conda.exe" | % { $_.FullName } + # make sure conda is on the path first in the path - - $Env:Path = 'C:\ProgramData\Miniconda3\Library\bin;' + $Env:Path - - $Env:Path = 'C:\ProgramData\Miniconda3\;' + $Env:Path - - $Env:Path = 'C:\ProgramData\Miniconda3\Scripts;' + $Env:Path + - $Env:Path = 'C:\ProgramData\miniforge3\;' + + 'C:\ProgramData\miniforge3\Scripts;' + + 'C:\ProgramData\miniforge3\Lib\bin;' + + $Env:PathOrig + - $Env:Path -split ';' - - echo "Our environemt vars:" - - $Env:Path + - python -c "import sys; print(sys.executable)" + + # This might be changing our current directory. It only needs to be run + # one time anyway, so we will comment it out. + #- Invoke-Expression "conda shell.powershell hook" + #- conda init powershell + + # On SO, it was suggested that this would perform a `conda init` + # without needing to restart PowerShell. So we will test it. + - C:\ProgramData\miniforge3\shell\condabin\conda-hook.ps1 - - echo $CI_PIPELINE_ID - - conda activate ./$CI_PIPELINE_ID - conda config --add channels defaults - conda config --add channels conda-forge - conda config --get channels - conda config --set channel_priority strict - script: - - echo "Building and testing PyGNOME" - - # # make sure our conda package is first in the path - # - $Env:Path = 'C:\ProgramData\Miniconda3\Library\bin;' + $Env:Path - # - $Env:Path = 'C:\ProgramData\Miniconda3\;' + $Env:Path - # - $Env:Path = 'C:\ProgramData\Miniconda3\Scripts;' + $Env:Path - - echo "Our environemt vars:" - - $Env:Path + - conda create -y -p ./pygnome python=$PYTHON_VER - - echo "Python is running from:" - - python -c "import sys; print(sys.executable)" + #- $Env:Path = '' + + # (Get-Location) + '\pygnome\;' + + # (Get-Location) + '\pygnome\Scripts\;' + + # (Get-Location) + '\pygnome\Lib\bin\;' + + # $Env:PathOrig + - conda activate ./pygnome + script: + - echo "Building and testing PyGNOME" # get the adios_db code: needed for tests and requirements - git clone --depth 1 -b develop https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.orr.noaa.gov/gnome/oil_database/oil_database.git - conda install -y python=$PYTHON_VER mamba + - cd ./py_gnome - echo "Setting up conda environment" - mamba install -y python=$PYTHON_VER --file conda_requirements.txt --file conda_requirements_build.txt --file conda_requirements_test.txt - --file oil_database/adios_db/conda_requirements.txt + --file ../oil_database/adios_db/conda_requirements.txt - - echo "Building PyGNOME" - - cd ./py_gnome - # - python setup.py install - - python setup.py develop - - # Make sure that it imports from some folder other than the - # gnome package folder. - # - cd ../ - # - python -c "import gnome" + # install adios_db + - pip install ../oil_database/adios_db/ + + # Install pygnome + # If you want verbose output including the compiler/linker commands, + # run the following command: + # python -m build --no-isolation --config-setting cmake.verbose=true + # + # And there may be times we want to build a wheel file (.whl) + # The following command will build a wheel in the current directory: + # python -m pip wheel ./ + - python -m pip install ./ - cd ./tests/unit_tests - pytest --runslow @@ -180,9 +209,9 @@ build_develop: # build the py_gnome image - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker build . --build-arg PYTHON_VER=$PYTHON_VER -t registry.orr.noaa.gov/gnome/pygnome:develop + - docker build . --build-arg PYTHON_VER=$PYTHON_VER -t registry.orr.noaa.gov/gnome/pygnome:$CI_COMMIT_BRANCH - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker push registry.orr.noaa.gov/gnome/pygnome:develop + - docker push registry.orr.noaa.gov/gnome/pygnome:$CI_COMMIT_BRANCH build_main: stage: build diff --git a/Install_without_conda.rst b/Install_without_conda.rst index 9c18e3df1..c633fb75e 100644 --- a/Install_without_conda.rst +++ b/Install_without_conda.rst @@ -28,7 +28,7 @@ Building everything by hand / with pip It is always good practice to create and run a complex system like this in a virtual environment of some sort: virtualenv, pipenv, etc. -A virtual environment is **not required** to run PyGNOME. +A virtual environment is *not required* to run PyGNOME. But you may be working in an environment (on a corporate network, for example) that restricts your access to the system files on your computer. In such a case, you may require a virtualenv in order to freely install python packages in python's site-packages dir. (site-packages is the standard place where python installers will put packages after building them) @@ -121,7 +121,7 @@ Download and install the newest Windows executable distribution of `Python 3.10 `_ -(3.9 -- 3.11 shoule work) +(3.9 -- 3.12 should work) A number of the packages that GNOME depends on have very complex and brittle build processes, and depend on third-party libraries that can be @@ -129,10 +129,6 @@ a challenge to build. Fortunately, many of them are available as binary wheels on PyPi -Another option is to use a Python scientific distribution, such as -`Anaconda `_ -- in which case, stop and read the ``Installing.rst`` file. - - Dependencies can be installed using the command:: > pip install -r requirements.txt @@ -152,25 +148,17 @@ Build PyGNOME > cd PyGNOME/py_gnome > pip install -r requirements.txt -3. Install the adios_db pacakge -- it is under active development along with py_gnome, so it's best to install that from source as well: +3. Install the adios_db package -- it is under active development along with py_gnome, so it's best to install that from source as well: https://github.com/NOAA-ORR-ERD/adios_oil_database/tree/production/adios_db -4. build the ``py_gnome`` module in develop or install mode: +4. build the ``py_gnome`` module in editable or install mode. - ``> python setup.py build`` + ``> python -m pip install ./`` or - ``> python setup.py install`` - - or - - ``> python setup.py develop`` - -The other option you may need is ``cleanall``, which should clean the development environment -- good to do after puling new code from git:: - - > python setup.py cleanall + ``> python -m pip install --editable ./`` 5. If this successfully completes, then run the unit tests:: diff --git a/Installing.rst b/Installing.rst index 6f9c1229c..9c44e6bd6 100644 --- a/Installing.rst +++ b/Installing.rst @@ -32,9 +32,7 @@ Activate the gnome environment:: Build the gnome package:: > cd py_gnome - > python setup.py develop - -.. note:: when you run this command, you will get a warning from ``setuptools`` that running ``setup.py`` directly is deprecated. That is expected, and the build system will be updated before it stops working altogether. + > python -m pip install ./ You now should be good to go, but to make sure: @@ -43,7 +41,8 @@ Run the tests:: > cd tests/unit_tests > pytest --runslow -NOTE: the "runslow" tests requiring downloading data for the tests -- you can eliminate that flag to get most of the tests to run without that delay. +NOTE: *The "runslow" tests requiring downloading data for the tests -- you can +eliminate that flag to get most of the tests to run without that delay.* All the details =============== @@ -51,37 +50,59 @@ All the details `Anaconda `__ is a Python distribution that has most of the difficult-to-build packages that PyGNOME needs already built in. Thus it's a nice target for running -GNOME on your own system. "conda" is the packaging manager used to manage the system. - -PyGNOME CAN be used with any Python distribution, but you will need to find or build a number of packages that are not easy to manage. If you are familiar with complex python packaging, then you can probably make it work. But conda makes it much easier, and that's what we use ourselves, and support that use. +GNOME on your own system. "conda" is the packaging manager used to +manage the system. + +PyGNOME can "in theory" be used with any Python distribution, and if you are +familiar with complex python packaging, then you might be able to make it work. +But you will need to find or build a number of dependent packages that are not +easy to manage. Additionally, for compiling our C/C++ components, we have +moved to a build system called +`scikit-build-core `__. +And it **will be quite difficult** to replicate the steps that are built into +this package. In short, we don't recommend it. Anaconda vs miniconda vs miniforge: ----------------------------------- -`Anaconda `__ provides a fairly complete python system for computational programming -- it is a large install, but comes with a lot of nice stuff pre-packaged that all works together. +`Anaconda `__ provides a fairly +complete python system for computational programming -- it is a large install, +but comes with a lot of nice stuff pre-packaged that all works together. -`miniconda `__ is a much smaller install -- it provides only Python and the conda package management system. You can install miniconda, and then install only the packages you need to run PyGNOME. +`miniconda `__ is a much smaller +install -- it provides only Python and the conda package management system. +You can install miniconda, and then install only the packages you need +to run PyGNOME. -`miniforge `__ is similar to miniconda, but designed to work primarily with the conda-forge packages. As everything required for PyGNOME is available via conda-forge, and many are not available anywhere else, miniforge is the easiest way to get PyGNOME up and running. You can install miniforge, and then install only the packages you need to run PyGNOME. +`miniforge `__ is similar to +miniconda, but designed to work primarily with the conda-forge packages. +As everything required for PyGNOME is available via conda-forge, and many +are not available anywhere else, miniforge is the easiest way to get PyGNOME +up and running. You can install miniforge, and then install only the packages +you need to run PyGNOME. -Either of these should work fine with PyGNOME, as long as you create an independent environment for it. +Either of these should work fine with PyGNOME, as long as you create an +independent environment for it. **NOTES:** -PyGNOME requires Python version 3.9 or greater (currently 3.10 is currently used operationally -- 3.11 should work, but it not well tested) +PyGNOME requires Python version 3.9 or greater (version 3.10 is currently used +operationally -- 3.11 and 3.12 should work, but it not well tested) -Anaconda (and miniconda) can be installed in either single-user or multi-user mode: +Anaconda (and miniconda and miniforge) can be installed in either single-user or multi-user mode: https://docs.continuum.io/anaconda/install -We (and Anaconda) recommend single-user mode (Select an install for “Just Me”) -- that way, administrator privileges are not required for either initial installation or maintaining the system. +We (and Anaconda) recommend single-user mode (Select an install for “Just Me”) +-- that way, administrator privileges are not required for either initial +installation or maintaining the system. Windows: ........ -You need the Windows 64 bit Python 3 version. Installing with the -defaults works fine. You probably want to let it set the PATH for you -- -that's a pain to do by hand. +You need the Windows 64 bit Python 3 version. Installing with the defaults +works fine. You probably want to let it set the PATH for you -- that's a pain +to do by hand. OS-X: @@ -99,77 +120,75 @@ Linux: The Linux 64bit-python3.10 is the one to use. -We do not support 32 bit on any platform. +NOTE: *We do not support 32 bit on any platform.* conda ----- -`conda `__ is the package -manager that Anaconda is built on. So when working with Anaconda, or miniconda, you -use the conda package manager for installing conda packages. ``pip`` -can also be used within conda, but it's best to use use conda for as much as you can. +`conda `__ is the package manager +that Anaconda is built on. So when working with Anaconda, you use the conda +package manager for installing conda packages. ``pip`` can also be used +within conda, but it's best to use use conda for as much as you can. -As a rule, if you need a new package, you should try to conda install it, and then, if there is not conda package available, you can pip install it. +As a rule, if you need a new package, you should try to conda install it, +and then, if there is no conda package available, you can pip install it. We have made sure that every package you need for PyGNOME is available for conda. Conda-Forge ........... -Conda-Forge (https://conda-forge.org/) is a community project that supplies a huge number of packages for the conda package manager. We have tried to assure that everything you need to run PyGNOME is available via the conda-forge channel. +`Conda-Forge `__ is a community project that supplies +a huge number of packages for the conda package manager. We have tried to +assure that everything you need to run PyGNOME is available via the +conda-forge channel. Setting up .......... Install: `Anaconda `__ - or alternatively: `Miniconda `__ or `Miniforge `_ -Once you have Anaconda, Miniconda, or Miniforge installed, the rest of the instructions should be the same. +Once you have Anaconda, Miniconda, or Miniforge installed, the rest of the +instructions should be the same. Update your (new) system ........................ -Once you have a conda system installed, you should start by -getting everything up to date, as sometimes packages have been updated -since the installer was built. +Once you have a conda system installed, you should start by getting everything +up to date, as sometimes packages have been updated since the installer was built. -First, update the conda package manager itself: - -Enter the following on the command-line:: +First, update the conda package manager itself. Enter the following on the command-line:: > conda update conda Setting up anaconda.org channels ................................ -`anaconda.org `__ is a web service for hosting conda packages for download. -The way this is done is through anaconda "channels", which can be thought of simply as places on ``anaconda.org`` where collections of packages are bundled together by the people hosting them. +`anaconda.org `__ is a web service for hosting +conda packages for download. The way this is done is through anaconda +"channels", which can be thought of simply as places on ``anaconda.org`` +where collections of packages are bundled together by the people hosting them. -Many of the dependencies that PyGNOME requires come out of the box -with Anaconda (or the conda "defaults" channel), but a few important -ones don't. **The "conda-forge" project:** -https://conda-forge.github.io/ - -Is a community project to build a wide variety of packages for conda -- -it should support everything that PyGNOME needs. +`conda-forge `__ is a community project to +build and make available a wide variety of packages for conda -- it should +support everything that PyGNOME needs. Adding another channel to conda: ................................ -To make it easy for your install to find conda-forge packages, it should be added to your conda configuration: - -Add the conda-forge channel:: +In order to find packages available on conda-forge, it should be added to your +conda channel configuration:: > conda config --add channels conda-forge -.. note:: MiniForge comes pre-configured to work with conda-forge, this step is not neccesary of you are using the miniforge distribution. +.. note:: MiniForge comes pre-configured to work with conda-forge, this step is not necessary if you are using the miniforge distribution. When you add a channel to conda, it puts it at the top of the list. So now when you install a package, conda will first look in conda-forge, @@ -188,9 +207,10 @@ In that order -- the order is important You also need to set the channel priority to "strict":: - > conda config --set channel_priority strict + > conda config --set channel_priority strict -This will assure that you will get packages from conda-forge, even if there are newer ones available in the defaults channel. +This will assure that you will get packages from conda-forge, even if there are +newer ones available in the defaults channel. conda environments ------------------ @@ -201,52 +221,54 @@ For more information see: http://conda.pydata.org/docs/using/envs.html -NOTE: We highly recommend that you use a conda environment for GNOME. +*NOTE: We highly recommend that you use a conda environment for GNOME.* -If you are only going to use Python / conda for PyGNOME, then you could use the base environment. -However, PyGNOME needs a number of specific package versions, so it is best to keep it separate from any other work you are doing. +If you are only going to use Python / conda for PyGNOME, then you could use +the base environment. However, there are a number of packages that PyGNOME +needs to be at specific versions, so it is best to keep it separate from +any other work you are doing. -(NOTE: you can do these steps with the Anaconda Navigator GUI if you have that installed) +*NOTE: You can do these steps with the Anaconda Navigator GUI if you have that +installed* Create an environment for PyGNOME:: > conda create -n gnome python=3.10 --file conda_requirements.txt --file conda_requirements_build.txt --file conda_requirements_test.txt -This will create an environment called "gnome" with Python itself and everything that it needs to be built, run, and tested -- it will be quite a bit, so it may take a while. +This will create an environment called "gnome" with Python itself and +everything that it needs to be built, run, and tested -- it will be quite a bit, +so it may take a while. To use that environment, you activate it with:: > conda activate gnome - and when you are done, you can deactivate it with:: > conda deactivate - After activating the environment, you can proceed with these instructions, -and all the packages PyGNOME needs will be available in that environment and kept separate from your main Anaconda install. +and all the packages PyGNOME needs will be installed into that environment +and kept separate from your main Anaconda install. You will need to activate the environment any time you want to work with PyGNOME in the future - Download the PyGNOME Code ------------------------- -PyGNOME is not currently available as a conda package, as it is under active development, and many users will need access to the source code. +PyGNOME is not currently available as a conda package, as it is under active +development, and many users will need access to the source code. Once you have a conda environment set up, you can compile and install PyGNOME. -You will need the files from the PyGNOME sources. If you -have not downloaded it yet, it is available here: +You will need the files from the PyGNOME sources. If you have not downloaded +it yet, it is available here: https://github.com/NOAA-ORR-ERD/PyGNOME You can either download a zip file of all the sources and unpack it, or -you can "clone" the git repository. If you clone the repository, you will -be able to update the code with the latest version with a simple command, -rather than having to re-download the whole package. +you can "clone" the git repository. Either choice is valid. Unless you want to contribute to the project, you should use the "main" branch. @@ -258,31 +280,36 @@ zip and tar archives of the PyGNOME source code can be found here: https://github.com/NOAA-ORR-ERD/PyGNOME/releases -This will get you the entire source archive of a given release, which is a fine way to work with PyGNOME. However, if you want to be able to quickly include changes as we update the code, you may want to work with a git "clone" of the source code instead. +This will get you the entire source archive of a given release, which is a +fine way to work with PyGNOME. However if, in the future, you want to use any +new changes that have been made to the code, you will need to re-download the +new release in its entirety. - -Cloning the PyGNOME git repository +Cloning the PyGNOME Git repository ---------------------------------- -git -... - -You will need a git client: +If you clone the repository, you will be able to update the code with the +latest version with a simple command (`git pull`). This will download only +the files that have changed and requires no archive extraction, so it will +not only be a faster operation, but we think you will find it is also more +convenient. -Linux: - it should be available from your package manager:: +First you will need a Git client. On Linux, it should be available from your +package manager using one of the following commands:: - > apt_get install git + > apt_get install git # Debian & Linux Mint or - > yum install git + > yum install git # CentOS & Red Hat Enterprise Linux -OS-X: - git comes with the XCode command line tools, which you will need for the compiler as well: +NOTE: *There are a few other Linux package managers out there. Look at this +`exhaustive list `__ +to find the one your Linux distribution uses* + +On OS-X, Git comes with the XCode command line tools: http://osxdaily.com/2014/02/12/install-command-line-tools-mac-os-x/ -Windows: - The "official" git for Windows installer is a good bet: +On Windows, the "official" Git for Windows installer is a good bet: https://git-for-windows.github.io/ @@ -290,45 +317,48 @@ Once you have the client, it's as easy as:: > git clone https://github.com/NOAA-ORR-ERD/PyGNOME.git -This will create a PyGNOME directory with all the code in it. +This will create a `./pygnome` directory with all the code in it. git branches: - git supports a number of different "branches" or versions of the code. You will most likley want to use the "main" branch (the default) unless you specifically want to experiment with a new feature. - + git supports a number of different "branches" or versions of the code. + You will most likley want to use the "main" branch (the default) unless you + specifically want to experiment with a new feature. Setting up conda ---------------- -If you have not already created an environment in which to run PyGNOME, follow the instructions above. +If you have not already created an environment in which to run PyGNOME, +follow the instructions above. To use the gnome environment you created, it needs to be activated with:: > conda activate gnome -and when you are done, you can deactivate it with:: - - > conda deactivate - -If you don't want to create an environment (or already have one), you can install what PyGNOME needs into an existing environment: +If you don't want to create an environment (or already have one), you can +install what PyGNOME needs into an existing environment:: -:: - - > cd PyGNOME # or wherever you put the PyGNOME project + > cd ./pygnome # or wherever you put the PyGNOME project > conda install --file conda_requirements.txt --file conda_requirements_build.txt --file conda_requirements_test.txt -NOTE: PyGNOME has a lot of specific dependencies -- it can be very hard for conda to resolve them with an large installed package base. If you have trouble, it's easiest to make a new environment just for PyGNOME. +NOTE: *PyGNOME has a lot of specific dependencies -- it can be very hard +for conda to resolve them with an large installed package base. +If you have trouble, it's easiest to make a new environment just for PyGNOME.* This should install all the packages required by PyGNOME. -(make sure you are in the correct conda environment, and you have the -conda-forge channel enabled) +(*make sure you are in the correct conda environment, and you have the +conda-forge channel enabled*) If installing the conda_requirements.txt fails: ............................................... -If you get an error about a particular package not being able to be installed, then conda will not install ANY of the packages in the file. We try hard to make sure everything is available on conda-forge. If however, a package of that particular version is missing, you can try: +If you get an error about a particular package not being able to be installed, +then conda will not install ANY of the packages in the file. We try hard +to make sure everything is available on conda-forge. If however, a package +of that particular version is missing, here are some things you can try. -Edit the conda_requirements.txt file and comment out the offending package by putting a "#" at the start of the line:: +Edit the conda_requirements.txt file and comment out the offending package +by putting a "#" at the start of the line:: ... scipy>=0.17 @@ -337,9 +367,11 @@ Edit the conda_requirements.txt file and comment out the offending package by pu gsw>=3.0.3 ... -That will disable that particular package, and hopefully everything else will install. +That will disable that particular package, and hopefully everything else +will install. -You can then try installing the offending package without a version specification:: +You can then try installing the offending package without a version +specification:: > conda install libgd @@ -349,51 +381,59 @@ And it may work for you. The ADIOS Oil Database ---------------------- -If you want to use PyGNOME with "real oil", rather than inert particles, you will need NOAA's ``adios_db`` package from the ADIOS Oil Database Project: +If you want to use PyGNOME with "real oil", rather than inert particles, +you will need NOAA's ``adios_db`` package from the ADIOS Oil Database Project: https://github.com/NOAA-ORR-ERD/adios_oil_database -This will allow you to use the JSON oil data format downloadable from NOAA's ADIOS Oil Database web app: +This will allow you to use the JSON oil data format downloadable from NOAA's +ADIOS Oil Database web app: https://adios.orr.noaa.gov/ -The ``adios_db`` package is available on conda-forge, and should have been installed by the process above. If not, it can be installed with :: +The ``adios_db`` package is available on conda-forge, and should have been +installed by the process above. If not, it can be installed with :: > conda install adios_db -However, the adios_db package is also under active development along with PyGNOME, so if you are working with the develop branch of PyGNOME, you may need the latest version of adios_db as well. In which case, you are best off downloading the sources from GitHub and installing it from source -- similar to PyGNOME. +However, the adios_db package is also under active development along with +PyGNOME, so if you are working with the develop branch of PyGNOME, +you may need the latest version of adios_db as well. In which case, +you are best off downloading the sources from GitHub and installing it +from source -- similar to PyGNOME. The latest releases (of the same branch) of each should be compatible. -cloning the repository :: - - > git clone https://github.com/NOAA-ORR-ERD/adios_oil_database.git +To clone the repository:: -Installing its dependencies:: + > git clone https://github.com/NOAA-ORR-ERD/adios_oil_database.git - > cd adios_db - > conda install --file conda_requirements.txt +To install its dependencies:: + > cd ./oil_database/adios_db + > conda install --file conda_requirements.txt Installing the package:: - > pip install ./ + > pip install ./ (or ``pip install -e ./`` to get an "editable" version) -Testing the adios_db install. +Testing the adios_db install +............................ -If you run the PyGNOME tests after having installed ``adios_db``, it will run a few additional tests that require the ``adios_db``. It should not need independent testing. +If you run the PyGNOME tests after having installed ``adios_db``, it will run +a few additional tests that require the ``adios_db``. It should not need +independent testing. But if you want to test it directly, you will need additional requirements:: > conda install --file conda_requirements_test.txt -And then you can run the tests: +And then you can run the tests:: > pytest --pyargs adios_db - Compilers --------- @@ -407,10 +447,9 @@ The system compiler for OS-X is XCode. It can be installed from the App Store. Apple has changed the XCode install process a number of times over the years. - -Rather than providing out of date information: - -You need the "Xcode Command Line Tools" -- look for Apple's documentation for how to install those. +Rather than providing out-of-date information, we will simply state that you +need the "Xcode Command Line Tools" -- look for Apple's documentation for +how to install those. Once the command line tools are installed, you should be able to build PyGNOME as described below. @@ -419,21 +458,33 @@ PyGNOME as described below. Windows ....... -For compiling python extensions on Windows with python3 it is best to use the Microsoft the Visual Studio 2019 (or later) Build Tools. They should be available here: +For compiling python extensions on Windows with python3 it is best to use +the Microsoft the Visual Studio 2019 (or later) Build Tools. They should be +available here: https://visualstudio.microsoft.com/downloads/ The free "Community" version should be fine. -Once installed, you will want to use one of the "Visual Studio Developer Command Prompts" provided to actually build PyGNOME -- it sets up the compiler for you. - -.. warning:: On some locked down systems, such as those at NOAA, the standard way to use the MS compiler will not work for a user that does not have administration privileges. - If you get errors about not being able to run the ``vcvarsall.bat`` script, then the compiler must be run as an administrator. - - If you have access to the NOAA/ORR GitLab server, a work around is supplied here: `Building Python extensions on Windows `_. - - If you have this issue and are not from NOAA, ask for help on Python fora or as an issue in the PyGNOME gitHub project. - +Once Visual Studio is installed, a number of +"Visual Studio Developer Command Prompt" applications will be made available +on the Windows toolbar. ``Scikit-build-core`` claims that it can intelligently +configure its environment to correctly build your package, but to be on the +safe side, you will want to open up the one with a name that looks something +similar to **"x64 Native Tools Command Prompt (for VS 20XX)"** in order to +build PyGNOME -- this is to make sure the compiler is setup for building +x64 targets. + +Warning: + On some locked down systems, such as those at NOAA, the + standard way to use the MS compiler will not work for a user that does not + have administration privileges. If you get errors about not being able to + run the ``vcvarsall.bat`` script, then the compiler must be run as an + administrator. If you have access to the NOAA/ORR GitLab server, a + work around is supplied here: + `Building Python extensions on Windows `__. + If you have this issue and are not from NOAA, ask for help on the Python + forum or as an issue in the PyGNOME gitHub project. Linux ..... @@ -441,54 +492,47 @@ Linux Linux uses the GNU gcc compiler. If it is not already installed on your system, use your system package manager to get it. -- apt for Ubuntu and Linux Mint -- rpm for Red Hat -- dpkg for Debian +- apt for Debian based distros (Ubuntu, Mint, Kali, ...) - yum for CentOS -- ??? for other distros +- `... `__ Building PyGNOME ................ -At this point you should have all the necessary third-party -tools in place. - -And it is probably best to build the "develop" target for your PyGNOME package if you plan on developing or debugging the PyGNOME source code -(or updating the source code from GitHub). - -Building the "develop" target allows changes in the python code -to be immediately available in your python environment without re-installing. +At this point you should have all the necessary third-party tools in place, +and you can build the PyGNOME package itself. But How you build the package +depends on how you plan to use it. -Of course if you plan on simply using the package, you may certainly -build with the "install" target. Just keep in mind that any updates to -the project will need to be rebuilt and re-installed in order for -changes to take effect. +Most people will likely want to simply use the package for building and running +simulations. For this, run the following:: -There are a number of options for building: + > cd /py_gnome + > python -m pip install ./ -:: +Just keep in mind that any updates to the project will need to be +rebuilt and re-installed in order for changes to take effect. - > python setup.py develop +*NOTE: You may have noticed that we run the pip module inside python instead of +running the `pip` executable directly. We have noticed on some platforms +(Windows) that conda virtual environments, when activated, sometimes don't +properly update the $Path environmental variable, causing pip to be run from +the base conda environment instead of the current one. The result is that +PyGNOME gets installed there instead of our current conda environment. +Running pip as a module ensures we are referencing the correct environment +for installation* -builds and installs the ``gnome`` package in "development" (editable) mode. +If you are planning to develop or debug the PyGNOME source code itself, +then you may want to perform a "editable" install. A "editable" install +puts a links intot he source code, rather than copying it into the Python install, so that changes in the python code are immediately available in your python environment without re-installing. -:: +For this, run the following:: - > python setup.py install + > cd /py_gnome + > python -m pip install --editable ./ -builds and installs the ``gnome`` package into your Python install. +If you would like or need to uninstall the package, run the following:: -:: - - > python setup.py cleanall - -cleans files generated by the build as well as files auto-generated by -cython. It is a good idea to run ``cleanall`` after updating from the -gitHub repo -- particularly if strange errors are occurring. - -You will need to re-run ``develop`` or ``install`` after running ``cleanall`` - -NOTE: PyGNOME is not currently configured to be built with pip -- you need to call ``setup.py`` directly. Note that you will get a message about setup.py being deprecated -- it will still work, and we will updated the build system soon. + > python -m pip uninstall gnome Testing PyGNOME @@ -499,7 +543,7 @@ PyGNOME is working properly. To run the tests:: - > cd py_gnome/tests/unit_tests + > cd /py_gnome/tests/unit_tests > pytest and if those pass, you can run:: @@ -517,10 +561,18 @@ runs. What if some tests fail? ........................ -We do our best to keep all tests passing on release versions of the package. But sometimes tests will fail due to the setup of the machine they are being run on -- package versions, etc. So the first thing to do is to make sure you have installed the dependencies as specified. +We do our best to keep all tests passing on release versions of the package. +But sometimes tests will fail due to the setup of the machine they are being +run on -- package versions, etc. So the first thing to do is to make sure you +have installed the dependencies as specified. -But ``gnome`` is large package -- hardly anyone is going to use all of it. So while we'd like all tests to pass, a given test failure may not be an issue for any given use case. -It's a bit hard to know whether a given test failure will affect your use case, but if you look at the name of the tests that fail, you might get a hint. For example, if any of the tests fail under ``test_weathering``, and you are not doing any oil weathering modeling, you don't need to worry about it. +But ``gnome`` is large package -- hardly anyone is going to use all of it. +So while we'd like all tests to pass, a given test failure may not be an issue +for any given use case. It's a bit hard to know whether a given test failure +will affect your use case, but if you look at the name of the tests that fail, +you might get a hint. For example, if any of the tests fail under +``test_weathering``, and you are not doing any oil weathering modeling, +you don't need to worry about it. In any case, you can try to run your use case, and see what happens. @@ -531,21 +583,24 @@ Running scripts There are a number of scripts in the ``scripts`` directory. -In ``example_scripts`` you will find examples of using the ``gnome`` package for various tasks. - -In ``testing_scripts`` you will find scripts that have been developed to test various features of the model. There are many more of these, so do look to see if they have what you need. But they are generally written in a less compact way as they are designed to exercise particular features. +In ``example_scripts`` you will find examples of using the ``gnome`` package +for various tasks. -You should be able to run these scripts in the same way as any Python script (with an IDE such as Spyder or PyCharm, or at the command line). +In ``testing_scripts`` you will find scripts that have been developed to +test various features of the model. There are many more of these, so do look +to see if they have what you need. But they are generally written in a +less compact way as they are designed to exercise particular features. +You should be able to run these scripts in the same way as any Python script +(with an IDE such as Spyder or PyCharm, or at the command line). -To run a script on the command line: -:: +To run a script on the command line:: > cd py_gnome/scripts/example_scripts -If you are using a conda environment: +If you are using a conda environment:: > conda activate gnome @@ -553,16 +608,13 @@ Run the script:: > python example_script.py -Each of the scripts exercises different features of PyGNOME -- they are hopefully well commented to see how they work. +Each of the scripts exercises different features of PyGNOME -- they are +hopefully well commented to see how they work. -In the ``testing_scripts`` dir, there is a ``run_all.py`` script that will run all the testing scripts -- primarily to make sure they all can still run as we update the model. +In the ``testing_scripts`` dir, there is a ``run_all.py`` script that will +run all the testing scripts -- primarily to make sure they all can still run +as we update the model. For further documentation of PyGNOME, see: https://gnome.orr.noaa.gov/doc/PyGNOME/index.html - - - - - - diff --git a/LICENSE.txt b/LICENSE.txt index eb810dfdf..604a22710 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -27,4 +27,4 @@ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -For more information, please refer to +For more information, please refer to diff --git a/README.rst b/README.rst index ec7da845f..e25712163 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,10 @@ Introduction developed by the National Oceanic and Atmospheric Administration (**NOAA**), Office of Response and Restoration (**ORR**), Emergency Response Division. -It is designed to support oil and other hazardous material spills in the coastal environment, and is also a full featured, flexible particle tracking system, that can be used for other oceanographic transport applications, such as fish larvae, marine debris, etc. +It is designed to support oil and other hazardous material spills in the +coastal environment, and is also a full featured, flexible particle tracking +system, that can be used for other oceanographic transport applications, +such as fish larvae, marine debris, etc. PyGNOME is a python package that encapsulates GNOME's functionality. @@ -38,25 +41,20 @@ Installation ============ We have put some effort into making this package reasonably easy, -or at least possible, to install in a number of ways on a few different computing platforms: +or at least possible, to install on a few different computing platforms: - OS-X - Windows - Linux (tested on CentOS 7) -At this time, it must be compiled from the source code. - -The "tricky" part is installing the dependencies: details in the following: - -`Install using conda <./Installing.rst>`_ - -The conda package manager is built primarily for scientific, engineering, -and math applications it is the easiest way to get set up to use ``PyGNOME``, and it is used in our development and testing process. - -If you don't want to / can't use conda -- here are some notes on that: - -`Installing Without Conda <./Install_without_conda.rst>`_ +This package contains modules written in C/C++, and they must be +compiled for this package to function, and we primarily use the Anaconda +distribution of Python for installation. Anaconda is built primarily for +scientific, engineering, and math applications. +It is now the only supported way to get set up to use ``PyGNOME``, +and it is used in our development and testing process. +`Install using Anaconda <./Installing.rst>`_ Compiling PyGnome ================= @@ -96,13 +94,15 @@ The WebGNOME Interface: ======================= Scripting is the most featurefull way to access PyGNOME's capabilities. -However we have developed a system that allows a user to create and run PyGNOME models from a web browser. +However we have developed a system that allows a user to create and run +PyGNOME models from a web browser. There is a publicly available instance of WebGNOME at: https://gnome.orr.noaa.gov -If you want to run your own instance of WebGNOME, the code is in the following projects: +If you want to run your own instance of WebGNOME, the code is in the +following projects: - `WebGnomeAPI `_: A web server that implements the PyGNOME interface @@ -111,4 +111,5 @@ If you want to run your own instance of WebGNOME, the code is in the following p **Fair Warning:** -The WebGNOME system is under active development, and by its very nature does not expose the full capabilities of PyGNOME. +The WebGNOME system is under active development, and by its very nature +does not expose the full capabilities of PyGNOME. diff --git a/dockerfile b/dockerfile index 0d6a50c97..18d4ecac3 100644 --- a/dockerfile +++ b/dockerfile @@ -1,25 +1,25 @@ ARG PYTHON_VER FROM registry.orr.noaa.gov/erd/centos-conda/miniforge-python$PYTHON_VER -# Args declared before the FROM need to be redeclared, don't remove this line +# Args declared before the FROM need to be redeclared, don't delete this ARG PYTHON_VER -RUN yum install -y wget gcc make bzip2 gcc-c++ ca-certificates \ - libglib2.0-0 libxext6 libsm6 libxrender1 \ - git mercurial subversion tar +RUN yum install -y libglib2.0-0 libxext6 libsm6 libxrender1 \ + wget gcc make bzip2 gcc-c++ chrpath patchelf \ + ca-certificates git mercurial subversion tar COPY ./ /pygnome/ WORKDIR /pygnome/ RUN conda install python=$PYTHON_VER \ - --file conda_requirements.txt \ - --file conda_requirements_build.txt \ + --file py_gnome/conda_requirements.txt \ + --file py_gnome/conda_requirements_build.txt \ --file oil_database/adios_db/conda_requirements.txt -RUN cd py_gnome && python setup.py install RUN cd oil_database/adios_db && python -m pip install ./ -# to check if they got installed properly -RUN python -c "import adios_db" -RUN python -c "import gnome" +RUN cd py_gnome && python -m pip install ./ +# to check if they got installed properly +#RUN python -c "import adios_db" +#RUN python -c "import gnome" diff --git a/py_gnome/CMakeLists.txt b/py_gnome/CMakeLists.txt new file mode 100644 index 000000000..aeb2ffde5 --- /dev/null +++ b/py_gnome/CMakeLists.txt @@ -0,0 +1,182 @@ +# to jcfr@github henryiii + + +cmake_minimum_required(VERSION 3.15...3.28) +project(${SKBUILD_PROJECT_NAME} LANGUAGES C CXX) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +find_package(NetCDF REQUIRED MODULE) + +find_package( + Python + COMPONENTS Interpreter Development.Module NumPy + REQUIRED +) + +find_program(CYTHON "cython") + +# On Windows, we need to set these environmental variables so that the linker +# can properly find the .lib file associated with a dynamic library (.dll). +# And they must be set before any add_library() commands +if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) + set(BUILD_SHARED_LIBS TRUE) +endif() + +set(EXT_NAMES + cy_basic_types + cy_mover + cy_helpers + cy_wind_mover + cy_current_mover + cy_cats_mover + cy_component_mover + cy_gridcurrent_mover + cy_gridwind_mover + cy_ice_mover + cy_ice_wind_mover + cy_currentcycle_mover + cy_ossm_time + cy_random_mover + cy_random_mover_3d + cy_rise_velocity_mover + cy_land_check + cy_grid_map + cy_shio_time + cy_grid + cy_grid_rect + cy_grid_curv + cy_weatherers +) + +set(LIBGNOME_CPP_FILES + RectGridVeL_c.cpp + MemUtils.cpp + Mover_c.cpp + Replacements.cpp + ClassID_c.cpp + Random_c.cpp + TimeValuesIO.cpp + GEOMETRY.cpp + OSSMTimeValue_c.cpp + TimeValue_c.cpp + RectUtils.cpp + WindMover_c.cpp + CompFunctions.cpp + StringFunctions.cpp + OUTILS.cpp + CATSMover_c.cpp + CurrentMover_c.cpp + ComponentMover_c.cpp + ShioTimeValue_c.cpp + ShioHeight.cpp + TriGridVel_c.cpp + DagTree.cpp + DagTreeIO.cpp + ShioCurrent1.cpp + ShioCurrent2.cpp + GridCurrentMover_c.cpp + GridWindMover_c.cpp + IceMover_c.cpp + IceWindMover_c.cpp + CurrentCycleMover_c.cpp + TimeGridVel_c.cpp + TimeGridWind_c.cpp + MakeTriangles.cpp + MakeDagTree.cpp + GridMap_c.cpp + GridMapUtils.cpp + RandomVertical_c.cpp + RiseVelocity_c.cpp + Weatherers_c.cpp +) + +list(TRANSFORM LIBGNOME_CPP_FILES PREPEND "lib_gnome/") + +add_library(lib_gnome SHARED ${LIBGNOME_CPP_FILES}) + +target_link_libraries(lib_gnome PRIVATE NetCDF::NetCDF) + +target_compile_definitions(lib_gnome PUBLIC pyGNOME=1) + +target_include_directories(lib_gnome PUBLIC lib_gnome) + +install(TARGETS lib_gnome LIBRARY DESTINATION gnome/cy_gnome) + +foreach(ext IN LISTS EXT_NAMES) + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/gnome/cy_gnome/${ext}.cpp" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/gnome/cy_gnome/${ext}.pyx" + VERBATIM + COMMAND "${CYTHON}" --cplus -3 + "${CMAKE_CURRENT_SOURCE_DIR}/gnome/cy_gnome/${ext}.pyx" + --output-file + "${CMAKE_CURRENT_BINARY_DIR}/gnome/cy_gnome/${ext}.cpp" + ) + + python_add_library( + ${ext} + MODULE "${CMAKE_CURRENT_BINARY_DIR}/gnome/cy_gnome/${ext}.cpp" + WITH_SOABI + ) + + target_link_libraries(${ext} PRIVATE lib_gnome Python::NumPy) + + + if(APPLE) + set_target_properties(${ext} PROPERTIES INSTALL_RPATH "@loader_path/.") + elseif(UNIX) + set_target_properties(${ext} PROPERTIES INSTALL_RPATH "$ORIGIN/.") + endif() + + install(TARGETS ${ext} DESTINATION gnome/cy_gnome) + +endforeach() + + +# +# Extension: cy_point_in_polygon +# + +# Now we make the c_point_in_polygon.c file into a library +# This will have to do until we figure out how to make a .pyx extension with +# additional .c files +# No need to make it dynamic since it +set(POINT_IN_POLY_C_FILES + c_point_in_polygon.c +) + +list(TRANSFORM POINT_IN_POLY_C_FILES PREPEND "gnome/utilities/geometry/") + +add_library(point_in_polygon SHARED ${POINT_IN_POLY_C_FILES}) + +install(TARGETS point_in_polygon DESTINATION gnome/utilities/geometry) + +# now we make the cy_point_in_polygon extension +add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/gnome/utilities/geometry/cy_point_in_polygon.c" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/gnome/utilities/geometry/cy_point_in_polygon.pyx" + VERBATIM + COMMAND "${CYTHON}" -3 -v + "${CMAKE_CURRENT_SOURCE_DIR}/gnome/utilities/geometry/cy_point_in_polygon.pyx" + --output-file + "${CMAKE_CURRENT_BINARY_DIR}/gnome/utilities/geometry/cy_point_in_polygon.c" +) + +python_add_library( + cy_point_in_polygon + MODULE "${CMAKE_CURRENT_BINARY_DIR}/gnome/utilities/geometry/cy_point_in_polygon.c" + WITH_SOABI +) + +target_link_libraries(cy_point_in_polygon PRIVATE point_in_polygon Python::NumPy) + +if(APPLE) + set_target_properties(cy_point_in_polygon PROPERTIES INSTALL_RPATH "@loader_path/.") +elseif(UNIX) + set_target_properties(cy_point_in_polygon PROPERTIES INSTALL_RPATH "$ORIGIN/.") +endif() + +install(TARGETS cy_point_in_polygon DESTINATION gnome/utilities/geometry) diff --git a/py_gnome/LICENSE.txt b/py_gnome/LICENSE.txt new file mode 100644 index 000000000..604a22710 --- /dev/null +++ b/py_gnome/LICENSE.txt @@ -0,0 +1,30 @@ +This work was created by an agency of the United States Federal Government. +As such, it is released into the Public Domain. + +Neither the Federal Government nor NOAA makes any guarantee as to its +suitability for any purpose. + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/py_gnome/MANIFEST.in b/py_gnome/MANIFEST.in new file mode 100644 index 000000000..6fa8c3d19 --- /dev/null +++ b/py_gnome/MANIFEST.in @@ -0,0 +1 @@ +prune utility_scripts diff --git a/py_gnome/README.rst b/py_gnome/README.rst new file mode 100644 index 000000000..880302cb5 --- /dev/null +++ b/py_gnome/README.rst @@ -0,0 +1,84 @@ + +.. image:: graphics/new_gnome_icon/GNOME_logo_450px-wide.png + :alt: GNOME Logo + :align: center + +####### +PyGNOME +####### + + +Introduction +============ + +**GNOME** (General NOAA Operational Modeling Environment) is a modeling tool +developed by the National Oceanic and Atmospheric Administration (**NOAA**), +Office of Response and Restoration (**ORR**), Emergency Response Division. + +It is designed to support oil and other hazardous material spills in the coastal environment, and is also a ful featured, flexible particle tracking system, that can be used for other oceangraphic transport applications, such as fish larvae, maringe debris, etc. + +PyGNOME is a python package that encapsulates GNOME's functionality. + +Disclaimer: +----------- + +**This code is under active development** + +* It should not be considered an officially endorsed NOAA product. +* Output produced by this code should not be considered endorsed by NOAA. + +Documentation +============= + +`Project Documentation `_ + +`FAQ `_ + + +Installation +============ + +We have put some effort into making this package reasonably easy, +or at least possible, to install in a number of ways on a few different computing platforms: + + - OS-X + - Windows + - Linux (tested on CentOS 7) + +At this time, it must be compiled from the source code. + +The "tricky" part is installing the dependencies: details in the following: + +`Install using conda <./Installing.rst>`_ + +The conda package manager is built primarily for scientific, engineering, +and math applications it is the easiest way to get set up to use ``PyGNOME``, and it what is used in our development and testing process. + +If you don't want to / can't use conda -- here are some notes on that: + +`Installing Without Conda <./Install_without_conda.rst>`_ + + +The WebGNOME Interface: +======================= + +Scripting is the most featureful way to access PyGNOME's capabilities. +However we have developed a system that allows a user to create and run PyGNOME models from a web browser. + +There is a publicly available instance of WebGNOME at: + +https://gnome.orr.noaa.gov + +If you want to run your own instance of WebGNOME, the code is in the following projects: + +- `WebGnomeAPI `_: + A web server that implements the PyGNOME interface +- `WebGnomeClient `_: + A Web application for setting up and running PyGNOME models + +**Fair Warning:** + +The WebGNOME system is under active development, and by its very nature does not expose the full capabilities of PyGNOME. + + + diff --git a/py_gnome/cmake/FindNetCDF.cmake b/py_gnome/cmake/FindNetCDF.cmake new file mode 100644 index 000000000..194b1a0d3 --- /dev/null +++ b/py_gnome/cmake/FindNetCDF.cmake @@ -0,0 +1,131 @@ +#[==[ +Provides the following variables: + + * `NetCDF_FOUND`: Whether NetCDF was found or not. + * `NetCDF_INCLUDE_DIRS`: Include directories necessary to use NetCDF. + * `NetCDF_LIBRARIES`: Libraries necessary to use NetCDF. + * `NetCDF_VERSION`: The version of NetCDF found. + * `NetCDF::NetCDF`: A target to use with `target_link_libraries`. + * `NetCDF_HAS_PARALLEL`: Whether or not NetCDF was found with parallel IO support. +#]==] + +function(FindNetCDF_get_is_parallel_aware include_dir) + file(STRINGS "${include_dir}/netcdf_meta.h" _netcdf_lines + REGEX "#define[ \t]+NC_HAS_PARALLEL[ \t]") + string(REGEX REPLACE ".*NC_HAS_PARALLEL[ \t]*([0-1]+).*" "\\1" _netcdf_has_parallel "${_netcdf_lines}") + if (_netcdf_has_parallel) + set(NetCDF_HAS_PARALLEL TRUE PARENT_SCOPE) + else() + set(NetCDF_HAS_PARALLEL FALSE PARENT_SCOPE) + endif() +endfunction() + +# Try to find a CMake-built NetCDF. +find_package(netCDF CONFIG QUIET) +if (netCDF_FOUND) + # Forward the variables in a consistent way. + set(NetCDF_FOUND "${netCDF_FOUND}") + set(NetCDF_INCLUDE_DIRS "${netCDF_INCLUDE_DIR}") + set(NetCDF_LIBRARIES "${netCDF_LIBRARIES}") + set(NetCDF_VERSION "${NetCDFVersion}") + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(NetCDF + REQUIRED_VARS NetCDF_INCLUDE_DIRS NetCDF_LIBRARIES + VERSION_VAR NetCDF_VERSION) + + if (NOT TARGET NetCDF::NetCDF) + add_library(NetCDF::NetCDF INTERFACE IMPORTED) + if (TARGET "netCDF::netcdf") + # 4.7.3 + set_target_properties(NetCDF::NetCDF PROPERTIES + INTERFACE_LINK_LIBRARIES "netCDF::netcdf") + elseif (TARGET "netcdf") + set_target_properties(NetCDF::NetCDF PROPERTIES + INTERFACE_LINK_LIBRARIES "netcdf") + else () + set_target_properties(NetCDF::NetCDF PROPERTIES + INTERFACE_LINK_LIBRARIES "${netCDF_LIBRARIES}") + endif () + endif () + + FindNetCDF_get_is_parallel_aware("${NetCDF_INCLUDE_DIRS}") + # Skip the rest of the logic in this file. + return () +endif () + +find_package(PkgConfig QUIET) +if (PkgConfig_FOUND) + pkg_check_modules(_NetCDF QUIET netcdf IMPORTED_TARGET) + if (_NetCDF_FOUND) + # Forward the variables in a consistent way. + set(NetCDF_FOUND "${_NetCDF_FOUND}") + set(NetCDF_INCLUDE_DIRS "${_NetCDF_INCLUDE_DIRS}") + set(NetCDF_LIBRARIES "${_NetCDF_LIBRARIES}") + set(NetCDF_VERSION "${_NetCDF_VERSION}") + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(NetCDF + REQUIRED_VARS NetCDF_LIBRARIES + # This is not required because system-default include paths are not + # reported by `FindPkgConfig`, so this might be empty. Assume that if we + # have a library, the include directories are fine (if any) since + # PkgConfig reported that the package was found. + # NetCDF_INCLUDE_DIRS + VERSION_VAR NetCDF_VERSION) + + if (NOT TARGET NetCDF::NetCDF) + add_library(NetCDF::NetCDF INTERFACE IMPORTED) + set_target_properties(NetCDF::NetCDF PROPERTIES + INTERFACE_LINK_LIBRARIES "PkgConfig::_NetCDF") + endif () + + FindNetCDF_get_is_parallel_aware("${_NetCDF_INCLUDEDIR}") + # Skip the rest of the logic in this file. + return () + endif () +endif () + +find_path(NetCDF_INCLUDE_DIR + NAMES netcdf.h + DOC "netcdf include directories") +mark_as_advanced(NetCDF_INCLUDE_DIR) + +find_library(NetCDF_LIBRARY + NAMES netcdf + DOC "netcdf library") +mark_as_advanced(NetCDF_LIBRARY) + +if (NetCDF_INCLUDE_DIR) + file(STRINGS "${NetCDF_INCLUDE_DIR}/netcdf_meta.h" _netcdf_version_lines + REGEX "#define[ \t]+NC_VERSION_(MAJOR|MINOR|PATCH|NOTE)") + string(REGEX REPLACE ".*NC_VERSION_MAJOR *\([0-9]*\).*" "\\1" _netcdf_version_major "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_MINOR *\([0-9]*\).*" "\\1" _netcdf_version_minor "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_PATCH *\([0-9]*\).*" "\\1" _netcdf_version_patch "${_netcdf_version_lines}") + string(REGEX REPLACE ".*NC_VERSION_NOTE *\"\([^\"]*\)\".*" "\\1" _netcdf_version_note "${_netcdf_version_lines}") + set(NetCDF_VERSION "${_netcdf_version_major}.${_netcdf_version_minor}.${_netcdf_version_patch}${_netcdf_version_note}") + unset(_netcdf_version_major) + unset(_netcdf_version_minor) + unset(_netcdf_version_patch) + unset(_netcdf_version_note) + unset(_netcdf_version_lines) + + FindNetCDF_get_is_parallel_aware("${NetCDF_INCLUDE_DIR}") +endif () + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NetCDF + REQUIRED_VARS NetCDF_LIBRARY NetCDF_INCLUDE_DIR + VERSION_VAR NetCDF_VERSION) + +if (NetCDF_FOUND) + set(NetCDF_INCLUDE_DIRS "${NetCDF_INCLUDE_DIR}") + set(NetCDF_LIBRARIES "${NetCDF_LIBRARY}") + + if (NOT TARGET NetCDF::NetCDF) + add_library(NetCDF::NetCDF UNKNOWN IMPORTED) + set_target_properties(NetCDF::NetCDF PROPERTIES + IMPORTED_LOCATION "${NetCDF_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${NetCDF_INCLUDE_DIR}") + endif () +endif () diff --git a/py_gnome/cmake/Readme.txt b/py_gnome/cmake/Readme.txt new file mode 100644 index 000000000..dacff88b6 --- /dev/null +++ b/py_gnome/cmake/Readme.txt @@ -0,0 +1,10 @@ +Assortedstuff needed by CMake + +At the moment: + +The netcdf libfinder from kitware / VTK: + +https://github.com/Kitware/VTK/blob/master/CMake/FindNetCDF.cmake + +This should get included in scikit build at some point. + diff --git a/conda_requirements.txt b/py_gnome/conda_requirements.txt similarity index 98% rename from conda_requirements.txt rename to py_gnome/conda_requirements.txt index 12a6cd634..246359c56 100644 --- a/conda_requirements.txt +++ b/py_gnome/conda_requirements.txt @@ -24,7 +24,7 @@ # it'll probably work with 3.9 -- 3.11, but CI is testing with 3.10 -python>=3.9,<3.12 +python>=3.9,<=3.12 ## System libs colander <2 diff --git a/conda_requirements_build.txt b/py_gnome/conda_requirements_build.txt similarity index 71% rename from conda_requirements_build.txt rename to py_gnome/conda_requirements_build.txt index 9b0e8a4ce..5dead964a 100644 --- a/conda_requirements_build.txt +++ b/py_gnome/conda_requirements_build.txt @@ -3,4 +3,9 @@ cython=3.* setuptools>=62 # older versions uses different build dirs +setuptools-scm>=6.2 gitpython +python-build +cmake +ninja +scikit-build-core>=0.7.* diff --git a/conda_requirements_docs.txt b/py_gnome/conda_requirements_docs.txt similarity index 100% rename from conda_requirements_docs.txt rename to py_gnome/conda_requirements_docs.txt diff --git a/conda_requirements_test.txt b/py_gnome/conda_requirements_test.txt similarity index 100% rename from conda_requirements_test.txt rename to py_gnome/conda_requirements_test.txt diff --git a/py_gnome/documentation/source/file_formats/index.rst b/py_gnome/documentation/source/file_formats/index.rst index 0ec34e25f..70336c19a 100644 --- a/py_gnome/documentation/source/file_formats/index.rst +++ b/py_gnome/documentation/source/file_formats/index.rst @@ -8,6 +8,7 @@ GNOME File Formats maps winds netcdf + WCOFS_netcdf noaa_specific_currents scaling_current_patterns output_formats diff --git a/py_gnome/documentation/source/file_formats/sample_gnome_output.nc b/py_gnome/documentation/source/file_formats/sample_gnome_output.nc new file mode 100644 index 000000000..01ceb2f5c Binary files /dev/null and b/py_gnome/documentation/source/file_formats/sample_gnome_output.nc differ diff --git a/py_gnome/documentation/source/tech_notes.rst b/py_gnome/documentation/source/tech_notes.rst index 50f336525..9570bdabe 100644 --- a/py_gnome/documentation/source/tech_notes.rst +++ b/py_gnome/documentation/source/tech_notes.rst @@ -6,6 +6,31 @@ Assorted technical notes about GNOME and its algorithms and use. These are often responses to questions from users. +Error messages and fixes +========================= + +Here are some known errors and tips for fixing them. + +Datetime errors from using `xarray`: +------------------------------------ + + + + TypeError: '<' not supported between instances of 'int' and 'datetime.datetime' + +`xarray` converts time to a numpy.datetime64 format that isn't compatible with the `datetime.datetime` format that is used throughout PyGNOME. The fix is to convert the time array from `numpy.datetime64` format to a `datetime` format. This can be accomplished with appending `.values.astype('datetime64[s]').tolist()` to the `numpy.datetime64` array, e.g.: + + compatible_time = u_xr.time_counter.values.astype('datetime64[s]').tolist() + + + +Dimension errors from using `xarray`: +------------------------------------- + ValueError: dimensions ('y', 'x') must have the same length as the number of data dimensions, ndim=1 + +This error reflects PyGNOME's dislike of `xarray.DataArray` and is fixed by slapping `.values` at the end of all `xarray.DataArray`, e.g.: + + node_lon = lon_psi.values The role of mass in GNOME ========================= diff --git a/py_gnome/gnome/__init__.py b/py_gnome/gnome/__init__.py index 5238f7c80..65b294509 100644 --- a/py_gnome/gnome/__init__.py +++ b/py_gnome/gnome/__init__.py @@ -7,8 +7,9 @@ set up the logger to dump to console. """ - import sys +import os +import pathlib import logging import json @@ -21,10 +22,24 @@ # just so it will be in the namespace. from .gnomeobject import GnomeId, AddLogger -__version__ = '1.1.7' +__version__ = '1.1.8' + + +if os.name == 'nt': + # In Windows, we need to add the location of our lib_gnome.dll to the + # .dll search path. + here = getattr(sys, '_stdlib_dir', None) + if not here and hasattr(os, '__file__'): + here = os.path.dirname(os.__file__) + if here: + os.add_dll_directory( + pathlib.Path(here) / 'site-packages' / 'bin' + ) -# a few imports so that the basic stuff is there +# +# A few imports so that the basic stuff is there +# def check_dependency_versions(): """ @@ -54,7 +69,7 @@ def ver_check(required, installed): libs = [('gridded', '0.6.5', ''), ('nucos', '3.2.0', ''), ('py_gd', '2.2.0', ''), - ('adios_db', '1.1.1', 'Only required to use the ADIOS Database ' + ('adios_db', '1.2.0', 'Only required to use the ADIOS Database ' 'JSON format for oil data.') ] diff --git a/py_gnome/gnome/cy_gnome/__init__.py b/py_gnome/gnome/cy_gnome/__init__.py index ed5e86075..522e20d57 100644 --- a/py_gnome/gnome/cy_gnome/__init__.py +++ b/py_gnome/gnome/cy_gnome/__init__.py @@ -2,9 +2,4 @@ __init__.py for the gnome/cy_gnome package """ - - - - - -#from . import cy_basic_types # make sure it's imported, for the C++ lib +from . import cy_basic_types # make sure it's imported, for the C++ lib diff --git a/py_gnome/gnome/cy_gnome/cy_basic_types.pyx b/py_gnome/gnome/cy_gnome/cy_basic_types.pyx index c4edcbd3b..6dc140848 100644 --- a/py_gnome/gnome/cy_gnome/cy_basic_types.pyx +++ b/py_gnome/gnome/cy_gnome/cy_basic_types.pyx @@ -39,7 +39,6 @@ def enum(**enums): """ LE Status as an enum type """ - class oil_status(IntEnum): """ maps to the C enum @@ -53,15 +52,6 @@ class oil_status(IntEnum): on_tideflat = OILSTAT_ON_TIDEFLAT -# oil_status = dict(not_released=OILSTAT_NOTRELEASED, -# in_water=OILSTAT_INWATER, -# on_land=OILSTAT_ONLAND, -# off_maps=OILSTAT_OFFMAPS, -# evaporated=OILSTAT_EVAPORATED, -# to_be_removed=OILSTAT_TO_BE_REMOVED, -# on_tideflat=OILSTAT_ON_TIDEFLAT, -# ) - class numerical_methods(IntEnum): euler = EULER, rk4 = RK4 @@ -79,6 +69,7 @@ disperse status as an enum type # remove = REMOVE, # have_removed = HAVE_REMOVED) # + """ SpillType {FORECAST_LE = 1, UNCERTAINTY_LE = 2}; """ @@ -117,7 +108,3 @@ class ts_format(IntEnum): cdef Seconds temp seconds = type(temp) - - - - diff --git a/py_gnome/gnome/environment/gridded_objects_base.py b/py_gnome/gnome/environment/gridded_objects_base.py index 6106b1f7f..68269f8e0 100644 --- a/py_gnome/gnome/environment/gridded_objects_base.py +++ b/py_gnome/gnome/environment/gridded_objects_base.py @@ -127,15 +127,6 @@ class Grid_U(gridded.grids.Grid_U, GnomeId): _schema = GridSchema - def __init__(self, **kwargs): - super(Grid_U, self).__init__(**kwargs) - - #This is for the COOPS case, where their coordinates go from 0-360 starting at prime meridian - for lon in [self.node_lon,]: - if lon is not None and lon.max() > 180: - self.logger.warning('Detected longitudes > 180 in {0}. Rotating -360 degrees'.format(self.name)) - lon -= 360 - def draw_to_plot(self, ax, features=None, style=None): import matplotlib def_style = {'color': 'blue', diff --git a/py_gnome/gnome/maps/map.py b/py_gnome/gnome/maps/map.py index 1af2a1a12..d82e4dbb9 100644 --- a/py_gnome/gnome/maps/map.py +++ b/py_gnome/gnome/maps/map.py @@ -468,6 +468,7 @@ def __init__(self, center=(0.0, 0.0), distance=30000, bearing=90, :param bearing: The bearing the closest point on the shoreline is from the center. """ + warnings.warn("ParamMap is deprecated. Code is no longer being updated", DeprecationWarning) refloat_halflife = kwargs.pop('refloat_halflife', 1) self._refloat_halflife = refloat_halflife * 3600 self.build(center, distance, bearing, units) diff --git a/py_gnome/gnome/model.py b/py_gnome/gnome/model.py index 2d9ba0d2c..f36dfade7 100644 --- a/py_gnome/gnome/model.py +++ b/py_gnome/gnome/model.py @@ -152,6 +152,7 @@ class ModelSchema(ObjTypeSchema): #manual_weathering = SchemaNode(Bool(), save=False, update=True, test_equal=False, missing=drop) weathering_activated = SchemaNode(Bool(), save=True, update=True, test_equal=False, missing=drop) + run_backwards = SchemaNode(Bool()) class Model(GnomeId): ''' @@ -207,6 +208,7 @@ def __init__(self, uncertain_spills=[], #manual_weathering=False, weathering_activated=False, + run_backwards=False, **kwargs): ''' Initializes a model. @@ -286,6 +288,15 @@ def __init__(self, self.time_step = time_step # this calls rewind() ! self._reset_num_time_steps() + # for backwards run set duration and time_step to be negative + self._run_backwards = run_backwards + if self._run_backwards is True: + if self.duration.total_seconds() > 0: + self._duration = timedelta(seconds = -self._duration.total_seconds()) + if self.time_step is not None and self.time_step > 0: + self.time_step = -self.time_step + + # default is to zip save file self.zipsave = True @@ -484,6 +495,33 @@ def uncertain(self, uncertain_value): def uncertain_spills(self): return self.spills.to_dict().get('uncertain_spills', []) + @property + def run_backwards(self): + ''' + Run backwards attribute of the model. If flag is toggled, rewind model + ''' + return self._run_backwards + + @run_backwards.setter + def run_backwards(self, run_backwards): + ''' + Run backwards attribute of the model + if gets toggled change sign of time and duration + ''' + if self._run_backwards != run_backwards: + self._run_backwards = run_backwards # update run_backwards + if self._run_backwards is True: + if self.duration.total_seconds() > 0: + self._duration = timedelta(seconds = -self._duration.total_seconds()) + if self.time_step is not None and self.time_step > 0: + self.time_step = -self.time_step + else: + if self.duration.total_seconds() < 0: + self._duration = timedelta(seconds = abs(self._duration.total_seconds())) + if self.time_step is not None and self.time_step < 0: + self.time_step = abs(self.time_step) + self.rewind() + @property def cache_enabled(self): ''' @@ -1570,6 +1608,7 @@ def check_inputs(self): (msgs, isValid) = self.validate() someSpillIntersectsModel = False + isWeatherable = False num_spills = len(self.spills) if num_spills == 0: msg = '{0} contains no spills'.format(self.name) @@ -1624,8 +1663,14 @@ def check_inputs(self): msgs.append(self._warn_pre + msg) - if spill.release_time < self.start_time + self.duration: - someSpillIntersectsModel = True + + + if not self.run_backwards: + if spill.release_time < self.start_time + self.duration: + someSpillIntersectsModel = True + else: + if spill.release_time > self.start_time + self.duration: + someSpillIntersectsModel = True if ((spill.release_time > self.start_time and self.time_step > 0) or (spill.release_time < self.start_time and self.time_step < 0)): @@ -1654,6 +1699,7 @@ def check_inputs(self): if spill.substance.is_weatherable: + isWeatherable = True pour_point = spill.substance.pour_point if spill.substance.water is not None: @@ -1714,22 +1760,17 @@ def check_inputs(self): msg = ('Time step and duration must have the same sign: time step = {0} duration = {1} ' 'To run backwards they must both be negative.' .format(self.time_step,self.duration.total_seconds())) - #self.logger.warning(msg) # for now make this a warning - #msgs.append('warning: ' + self.__class__.__name__ + ': ' + msg) isValid = False raise GnomeRuntimeError(msg) if self.duration.total_seconds() > 0: msg = ('Time step and duration must have the same sign: time step = {0} duration = {1} ' 'To run backwards they must both be negative.' .format(self.time_step,self.duration.total_seconds())) - #self.logger.warning(msg) # for now make this a warning - #msgs.append('warning: ' + self.__class__.__name__ + ': ' + msg) isValid = False - if self.has_weathering: + if self.has_weathering and isWeatherable: + #if self.weathering_activated: # might have a better check for weathering msg = ('Backwards run is not valid for weathering. ' 'Turn off weathering to model a backwards trajectory.') - #self.logger.warning(msg) # for now make this a warning - #msgs.append('warning: ' + self.__class__.__name__ + ': ' + msg) isValid = False raise GnomeRuntimeError(msg) diff --git a/py_gnome/gnome/movers/movers.py b/py_gnome/gnome/movers/movers.py index d8ee62d4f..5e55a78ba 100644 --- a/py_gnome/gnome/movers/movers.py +++ b/py_gnome/gnome/movers/movers.py @@ -359,16 +359,23 @@ def prepare_for_model_step(self, sc, time_step, model_time_datetime): uncertain_spill_count, uncertain_spill_size) except OSError as e: - msg = ('No available data in the time interval ' - 'that is being modeled\n' - '\tModel time: {}\n' - '\tData available from {} to {}\n' - '\tMover: {} of type {}\n' - '\tError: {}' - .format(model_time_datetime, - self.data_start, self.data_stop, - self.name, self.__class__, - str(e))) + if "reference point" in str(e): + msg = ('Reference point not valid ' + '\tMover: {} of type {}\n' + '\tError: {}' + .format(self.name, self.__class__, + str(e))) + else: + msg = ('No available data in the time interval ' + 'that is being modeled\n' + '\tModel time: {}\n' + '\tData available from {} to {}\n' + '\tMover: {} of type {}\n' + '\tError: {}' + .format(model_time_datetime, + self.data_start, self.data_stop, + self.name, self.__class__, + str(e))) self.logger.error(msg) raise RuntimeError(msg) diff --git a/py_gnome/gnome/outputters/erma_data_package.py b/py_gnome/gnome/outputters/erma_data_package.py index dc48f339f..f61a7b050 100644 --- a/py_gnome/gnome/outputters/erma_data_package.py +++ b/py_gnome/gnome/outputters/erma_data_package.py @@ -23,6 +23,9 @@ from gnome.utilities import convert_mass_to_mass_or_volume from .outputter import Outputter, BaseOutputterSchema + +erma_data_package_data_dir = pathlib.Path(__file__).parent / "erma_data_package_data" + class ERMADataPackageSchema(BaseOutputterSchema): filename = FilenameSchema( missing=drop, save=True, update=True, test_equal=False @@ -414,9 +417,7 @@ def build_package(self): # Write out some empty dirs for now for attachments etc. zipf.writestr(zipfile.ZipInfo('attachment/'), '') # Needed font file - font_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'SHAPES.TTF') + font_path = erma_data_package_data_dir / 'SHAPES.TTF' zipf.write(font_path, arcname='support_files/fonts/SHAPES.TTF') zipf.close() @@ -429,13 +430,11 @@ def make_contour_polygon_package_layer(self, id, shapefile_filename, generic_name = 'contour_certain' generic_description = 'Contour Certain' layer_template = None - layer_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'layer_template.json') + + layer_template_path = erma_data_package_data_dir / 'layer_template.json' contour_template = None - contour_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'default_contour_template.json') + contour_template_path = erma_data_package_data_dir / 'default_contour_template.json' + with open(layer_template_path) as f: layer_template = json.load(f) with open(contour_template_path) as f: @@ -523,13 +522,9 @@ def make_boundary_polygon_package_layer(self, id, uncertain, shapefile_filename, generic_name = f'boundary_{"uncertain" if uncertain else "certain"}' generic_description = f'Boundary {"Uncertain" if uncertain else "Certain"}' layer_template = None - layer_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'layer_template.json') + layer_template_path = erma_data_package_data_dir / 'layer_template.json' polygon_template = None - polygon_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'default_polygon_cartoline_template.json') + polygon_template_path = erma_data_package_data_dir / 'default_polygon_cartoline_template.json' with open(layer_template_path) as f: layer_template = json.load(f) with open(polygon_template_path) as f: @@ -613,13 +608,10 @@ def make_map_polygon_package_layer(self, id, map_polygon_name, shapefile_name, shz_name = os.path.join(self.tempdir.name, shapefile_name+'.shz') shapefile_pathlib_path = pathlib.Path(shz_name) layer_template = None - layer_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'layer_template.json') + layer_template_path = erma_data_package_data_dir / 'layer_template.json' polygon_template = None - polygon_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'default_polygon_template.json') + polygon_template_path = erma_data_package_data_dir / 'default_polygon_template.json' + with open(layer_template_path) as f: layer_template = json.load(f) with open(polygon_template_path) as f: @@ -675,13 +667,9 @@ def make_spill_location_package_layer(self, id): shz_name = os.path.join(self.tempdir.name, shapefile_name+'.shz') shapefile_pathlib_path = pathlib.Path(shz_name) layer_template = None - layer_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'layer_template.json') + layer_template_path = erma_data_package_data_dir / 'layer_template.json' default_spill_location_template = None - default_spill_location_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'default_spill_location_template.json') + default_spill_location_template_path = erma_data_package_data_dir / 'default_spill_location_template.json' with open(layer_template_path) as f: layer_template = json.load(f) with open(default_spill_location_template_path) as f: @@ -816,15 +804,9 @@ def generate_cutoff_struct(self): def make_particle_package_layer(self, id, layer_name, uncertain, shapefile_filename): dir, basefile = os.path.split(shapefile_filename) output_path = dir+"/"+str(id)+".json" - layer_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'layer_template.json') - default_floating_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'default_floating_template.json') - default_beached_template_path = os.path.join(os.path.split(__file__)[0], - 'erma_data_package', - 'default_beached_template.json') + layer_template_path = erma_data_package_data_dir / 'layer_template.json' + default_floating_template_path = erma_data_package_data_dir / 'default_floating_template.json' + default_beached_template_path = erma_data_package_data_dir / 'default_beached_template.json' layer_template = None default_floating_template = default_beached_template = None with open(layer_template_path) as f: diff --git a/py_gnome/gnome/outputters/erma_data_package/SHAPES.TTF b/py_gnome/gnome/outputters/erma_data_package_data/SHAPES.TTF similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/SHAPES.TTF rename to py_gnome/gnome/outputters/erma_data_package_data/SHAPES.TTF diff --git a/py_gnome/gnome/outputters/erma_data_package/default_beached_template.json b/py_gnome/gnome/outputters/erma_data_package_data/default_beached_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/default_beached_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/default_beached_template.json diff --git a/py_gnome/gnome/outputters/erma_data_package/default_contour_template.json b/py_gnome/gnome/outputters/erma_data_package_data/default_contour_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/default_contour_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/default_contour_template.json diff --git a/py_gnome/gnome/outputters/erma_data_package/default_floating_template.json b/py_gnome/gnome/outputters/erma_data_package_data/default_floating_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/default_floating_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/default_floating_template.json diff --git a/py_gnome/gnome/outputters/erma_data_package/default_polygon_cartoline_template.json b/py_gnome/gnome/outputters/erma_data_package_data/default_polygon_cartoline_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/default_polygon_cartoline_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/default_polygon_cartoline_template.json diff --git a/py_gnome/gnome/outputters/erma_data_package/default_polygon_template.json b/py_gnome/gnome/outputters/erma_data_package_data/default_polygon_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/default_polygon_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/default_polygon_template.json diff --git a/py_gnome/gnome/outputters/erma_data_package/default_spill_location_template.json b/py_gnome/gnome/outputters/erma_data_package_data/default_spill_location_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/default_spill_location_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/default_spill_location_template.json diff --git a/py_gnome/gnome/outputters/erma_data_package/layer_template.json b/py_gnome/gnome/outputters/erma_data_package_data/layer_template.json similarity index 100% rename from py_gnome/gnome/outputters/erma_data_package/layer_template.json rename to py_gnome/gnome/outputters/erma_data_package_data/layer_template.json diff --git a/py_gnome/gnome/outputters/outputter.py b/py_gnome/gnome/outputters/outputter.py index bbd38ad52..484f3a863 100644 --- a/py_gnome/gnome/outputters/outputter.py +++ b/py_gnome/gnome/outputters/outputter.py @@ -174,10 +174,10 @@ def output_timestep(self, value): if value is None: self._output_timestep = None else: - if value == timedelta(0): - raise ValueError("output_timestep cannot be zero") - #if value <= timedelta(0): - #raise ValueError("output_timestep must be positive") + #if value == timedelta(0): + #raise ValueError("output_timestep cannot be zero") + if value <= timedelta(0): + raise ValueError("output_timestep must be positive") self._output_timestep = int(value.total_seconds()) def prepare_for_model_run(self, @@ -232,12 +232,12 @@ def prepare_for_model_run(self, self.sc_pair = spills if self._output_timestep is not None: - if self._output_timestep < 0 and model_time_step > 0: - warnings.warn(f"Outputter output timestep {self.output_timestep} is less than " - f"zero.", RuntimeWarning) - if self._output_timestep > 0 and model_time_step < 0: - warnings.warn(f"Outputter output timestep {self.output_timestep} is greater than " - f"zero.", RuntimeWarning) +# if self._output_timestep < 0 and model_time_step > 0: +# warnings.warn(f"Outputter output timestep {self.output_timestep} is less than " +# f"zero.", RuntimeWarning) +# if self._output_timestep > 0 and model_time_step < 0: +# warnings.warn(f"Outputter output timestep {self.output_timestep} is greater than " +# f"zero.", RuntimeWarning) if abs(self._output_timestep) < abs(model_time_step): warnings.warn(f"Outputter output timestep {self.output_timestep} is less than " f"model time step: {model_time_step} seconds. " @@ -302,7 +302,8 @@ def prepare_for_model_step(self, time_step, model_time): self._is_first_output = False return - if model_time + d > self.output_start_time: + if ((model_time + d > self.output_start_time and time_step > 0) or + (model_time + d < self.output_start_time and time_step < 0)): if self._is_first_output: self._write_step = True self._is_first_output = False diff --git a/py_gnome/gnome/spill_container.py b/py_gnome/gnome/spill_container.py index 5674f9e9c..74e95fb63 100644 --- a/py_gnome/gnome/spill_container.py +++ b/py_gnome/gnome/spill_container.py @@ -445,7 +445,7 @@ def _set_substancespills(self): "trying to add :{}\n" "These are the substances in the on spills:\n" "{}".format(substance, subs)) - + # set the number of oil components # fixme: with only one substance this could be determined elsewhere if hasattr(substance, 'num_components'): @@ -815,7 +815,8 @@ def prepare_for_model_run(self, array_types=None, time_step=300): #self._append_initializer_array_types(array_types) for s in self.spills: - s.prepare_for_model_run(time_step) + if s.on: + s.prepare_for_model_run(time_step) ats = default_array_types.copy() ats.update(array_types) self._array_types = ats diff --git a/py_gnome/gnome/spills/release.py b/py_gnome/gnome/spills/release.py index 090c7b00d..56f429787 100644 --- a/py_gnome/gnome/spills/release.py +++ b/py_gnome/gnome/spills/release.py @@ -350,6 +350,9 @@ def prepare_for_model_run(self, ts): ''' if self._prepared: self.rewind() + if ts < 1 and (self.end_release_time != self.release_time): + raise ValueError('Backwards run is not valid for continuous releases. \ + Use an instantaneous release or run forwards.') if self.LE_timestep_ratio(ts) < 1: raise ValueError('Not enough LEs: Number of LEs must at least \ be equal to the number of timesteps in the release') @@ -583,6 +586,9 @@ def initialize_LEs(self, to_rel, sc, start_time, end_time): ''' # if time_step == 0: # time_step = 1 # to deal with initializing position in instantaneous release case + if self.release_duration == 0 and start_time != end_time: # special case for instantaneous spill with release after model start + if self.release_time <= end_time and self.release_time>=start_time: + start_time = end_time = self.release_time if start_time == end_time: end_time += timedelta(seconds=1) sl = slice(-to_rel, None, 1) diff --git a/py_gnome/gnome/utilities/__init__.py b/py_gnome/gnome/utilities/__init__.py index bb8f1ccee..c2d38703d 100644 --- a/py_gnome/gnome/utilities/__init__.py +++ b/py_gnome/gnome/utilities/__init__.py @@ -40,7 +40,7 @@ def convert_longitude(lon, coord_system='-180--180'): -180 converts to 180 It should be safe to call this on any coords -- if they are already - in the expected format, they will not be changes, except for the + in the expected format, they will not be changed, except for the normalization above. """ diff --git a/py_gnome/gnome/utilities/geometry/setup.py b/py_gnome/gnome/utilities/geometry/setup.py index 032447ad8..9db13a651 100644 --- a/py_gnome/gnome/utilities/geometry/setup.py +++ b/py_gnome/gnome/utilities/geometry/setup.py @@ -3,16 +3,17 @@ """ setup.py for geometry package -It's now built with the main gnome setup.py, but kept this here for easier testing in place... -only useful now for "develop" mode +It's now built with the main gnome setup.py, but kept this here for +easier testing in place... +Only useful now for "develop" mode """ -#from distutils.core import setup -from setuptools import setup # to support "develop" mode +# from distutils.core import setup +from setuptools import setup # to support "develop" mode from distutils.extension import Extension from Cython.Distutils import build_ext -import numpy # for the includes for the Cython code +import numpy # for the includes for the Cython code ext_modules = [Extension("cy_point_in_polygon", @@ -22,19 +23,19 @@ ] setup( - name = "geometry", - version = "0.03", - author = "Chris Barker", - author_email = "Chris.Barker@noaa.gov", - description = ("some geometry for GNOME"), -# long_description=read('README'), + name="geometry", + version="0.03", + author="Chris Barker", + author_email="Chris.Barker@noaa.gov", + description=("some geometry for GNOME"), + # long_description=read('README'), classifiers=[ "Development Status :: 3 - Alpha", "Topic :: Utilities", ], - cmdclass = {'build_ext': build_ext}, -# packages = ["geometry"], - ext_modules = ext_modules, -# modules = ["BBox.py",] - scripts = [], + cmdclass={'build_ext': build_ext}, + # packages=["geometry"], + ext_modules=ext_modules, + # modules=["BBox.py",] + scripts=[], ) diff --git a/py_gnome/gnome/utilities/hull.py b/py_gnome/gnome/utilities/hull.py index 7b96f5072..405b0a4a6 100644 --- a/py_gnome/gnome/utilities/hull.py +++ b/py_gnome/gnome/utilities/hull.py @@ -1,46 +1,55 @@ import geopandas as gpd -import pandas as pd -import numpy as np from shapely.geometry import Polygon, MultiPolygon, Point, MultiPoint, LineString -from shapely import concave_hull, union_all, buffer +from shapely import concave_hull, union_all + # Buffer by a number of meters def buffer_hull(this_hull, buffer_distance=1): - schema = {'geometry':[this_hull]} - geodataframe = gpd.GeoDataFrame(schema, crs='epsg:4326', geometry='geometry') + schema = {'geometry': [this_hull]} + geodataframe = gpd.GeoDataFrame(schema, crs='epsg:4326', + geometry='geometry') geodataframe_3857 = geodataframe.to_crs(epsg=3857) geodataframe_3857_buffered = geodataframe_3857.buffer(buffer_distance) geodataframe_3857_buffered_back_to_4326 = geodataframe_3857_buffered.to_crs(epsg=4326) return geodataframe_3857_buffered_back_to_4326[0] + def calculate_hull(spill_container, ratio=0.5, allow_holes=False, separate_by_spill=False): hulls_found = [] if separate_by_spill: # Make a dataframe - schema = {'positions':[Point(point) for point in spill_container['positions']], - 'spill_num':spill_container['spill_num']} - data_frame = gpd.GeoDataFrame(schema, crs='epsg:4326', geometry='positions') + schema = {'positions': [Point(point) + for point in spill_container['positions']], + 'spill_num': spill_container['spill_num']} + data_frame = gpd.GeoDataFrame(schema, crs='epsg:4326', + geometry='positions') spill_nums = data_frame['spill_num'].unique() for spill_num in spill_nums: this_spill = data_frame[data_frame['spill_num'] == spill_num] - this_mpt = MultiPoint([Point(point) for point in this_spill['positions']]) - this_hull = concave_hull(this_mpt, ratio=ratio, allow_holes=allow_holes) - if (isinstance(this_hull, Point) or isinstance(this_hull, LineString)): + this_mpt = MultiPoint([Point(point) + for point in this_spill['positions']]) + this_hull = concave_hull(this_mpt, ratio=ratio, + allow_holes=allow_holes) + if (isinstance(this_hull, Point) or + isinstance(this_hull, LineString)): this_hull = buffer_hull(this_hull) if (isinstance(this_hull, Polygon) or - isinstance(this_hull, MultiPolygon)): + isinstance(this_hull, MultiPolygon)): hulls_found.append(this_hull) else: # We want a hull around everything - schema = {'positions':[Point(point) for point in spill_container['positions']]} + schema = {'positions': [Point(point) + for point in spill_container['positions']]} data_frame = gpd.GeoDataFrame(schema, crs='epsg:4326') - this_mpt = MultiPoint([Point(point) for point in spill_container['positions']]) - this_hull = concave_hull(this_mpt, ratio=ratio, allow_holes=allow_holes) + this_mpt = MultiPoint([Point(point) + for point in spill_container['positions']]) + this_hull = concave_hull(this_mpt, ratio=ratio, + allow_holes=allow_holes) if (isinstance(this_hull, Point) or isinstance(this_hull, LineString)): this_hull = buffer_hull(this_hull) if (isinstance(this_hull, Polygon) or - isinstance(this_hull, MultiPolygon)): + isinstance(this_hull, MultiPolygon)): hulls_found.append(this_hull) final_geom = union_all(hulls_found) return final_geom if (isinstance(final_geom, Polygon) or @@ -48,7 +57,7 @@ def calculate_hull(spill_container, ratio=0.5, allow_holes=False, # Cutoffs defined in a structure like the following (by spill_num): -#cutoff_struct = {1: {'param': 'mass', +# cutoff_struct = {1: {'param': 'mass', # 'cutoffs': [{'cutoff': 100, # 'label': 'low'}, # {'cutoff': 500, @@ -75,8 +84,9 @@ def calculate_contours(spill_container, cutoff_struct=None, # The spills requested are the keys spills = cutoff_struct.keys() # Make a dataframe - schema = {'positions':[Point(point) for point in spill_container['positions']], - 'spill_num':spill_container['spill_num'], + schema = {'positions': [Point(point) + for point in spill_container['positions']], + 'spill_num': spill_container['spill_num'], 'mass': spill_container['mass'], 'age': spill_container['age'], 'status_codes': spill_container['status_codes']} @@ -89,7 +99,8 @@ def calculate_contours(spill_container, cutoff_struct=None, schema['frac_water'] = spill_container['frac_water'] if 'density' in spill_container: schema['density'] = spill_container['density'] - data_frame = gpd.GeoDataFrame(schema, crs='epsg:4326', geometry='positions') + data_frame = gpd.GeoDataFrame(schema, crs='epsg:4326', + geometry='positions') for spill_num in spills: this_spill = data_frame[data_frame['spill_num'] == spill_num] this_struct = cutoff_struct[spill_num] @@ -99,19 +110,23 @@ def calculate_contours(spill_container, cutoff_struct=None, # First one... we just do anything less than the current cutoff this_contour_set = this_spill[this_spill[param] <= cutoff['cutoff']] elif idx == len(this_struct['cutoffs'])-1: - # Last one... we just do anything greater than the previous cutoff + # Last one... we just do anything greater than the + # previous cutoff this_contour_set = this_spill[this_spill[param] >= this_struct['cutoffs'][idx-1]['cutoff']] else: # Else we bracket it... this_contour_set = this_spill[this_spill[param] <= cutoff['cutoff']] this_contour_set = this_contour_set[this_contour_set[param] >= this_struct['cutoffs'][idx-1]['cutoff']] if len(this_contour_set['positions']) > 0: - this_contour_mpt = MultiPoint([Point(point) for point in this_contour_set['positions']]) - this_hull = concave_hull(this_contour_mpt, ratio=ratio, allow_holes=allow_holes) - if (isinstance(this_hull, Point) or isinstance(this_hull, LineString)): + this_contour_mpt = MultiPoint([Point(point) for point in + this_contour_set['positions']]) + this_hull = concave_hull(this_contour_mpt, ratio=ratio, + allow_holes=allow_holes) + if (isinstance(this_hull, Point) + or isinstance(this_hull, LineString)): this_hull = buffer_hull(this_hull) if (isinstance(this_hull, Polygon) or - isinstance(this_hull, MultiPolygon)): + isinstance(this_hull, MultiPolygon)): contours_found.append({'spill_num': spill_num, 'cutoff': cutoff['cutoff'], 'cutoff_id': cutoff['cutoff_id'], diff --git a/py_gnome/gnome/utilities/map_canvas.py b/py_gnome/gnome/utilities/map_canvas.py index 954d74ac9..73092346c 100644 --- a/py_gnome/gnome/utilities/map_canvas.py +++ b/py_gnome/gnome/utilities/map_canvas.py @@ -11,7 +11,6 @@ This should have the basic drawing stuff. Ideally nothing in here is GNOME-specific. """ - import bisect import numpy as np @@ -197,8 +196,8 @@ def add_colors(self, color_list): """ Add a list of colors to the pallette - :param color_list: list of colors. Each element of the list is a 2-tuple: - ('color_name', (r,g,b)) + :param color_list: list of colors. Each element of the list + is a 2-tuple: ('color_name', (r,g,b)) """ self.fore_image.add_colors(color_list) self.back_image.add_colors(color_list) @@ -207,7 +206,8 @@ def add_color_ramp(self, color_scheme, min_val, max_val): """ generate depth colors to the gd images :param color_scheme: color scheme to render images - type: 'magma', 'inferno', 'plasma', 'viridis', 'cividis', 'twilight', 'twilight_shifted', 'turbo' + type: 'magma', 'inferno', 'plasma', 'viridis', 'cividis', + 'twilight', 'twilight_shifted', 'turbo' :param min_val: value to map to the first color in the scheme :param max_val: value to map to the last color in the scheme """ @@ -215,12 +215,14 @@ def add_color_ramp(self, color_scheme, min_val, max_val): # print(existing_colors) - cr = py_gd.color_ramp.ColorRamp(color_scheme, min_val, max_val, base_colorscheme=len(existing_colors)) + cr = py_gd.color_ramp.ColorRamp(color_scheme, + min_val, max_val, + base_colorscheme=len(existing_colors)) self.fore_image.add_colors(cr.colorlist) self.back_image.add_colors(cr.colorlist) - self._color_ramp=cr + self._color_ramp = cr def get_color_names(self): """ @@ -441,6 +443,9 @@ def save(self, filename, file_type='png'): This copies the foreground image on top of the background image and saves the whole thing. + Fixme: filename & file_type appear to not be used at all here. + Are we passing them in to satisfy some api specification? + :param filename: full path of file to be saved to :type filename: string diff --git a/py_gnome/gnome/utilities/orderedcollection.py b/py_gnome/gnome/utilities/orderedcollection.py index 6a2b90ccb..0bfc136da 100644 --- a/py_gnome/gnome/utilities/orderedcollection.py +++ b/py_gnome/gnome/utilities/orderedcollection.py @@ -1,5 +1,6 @@ #!/usr/bin/env python + class OrderedCollection(object): ''' Generalized Container for a set of objects of a particular type which @@ -14,22 +15,22 @@ class OrderedCollection(object): def __init__(self, elems=None, dtype=None): if elems and not isinstance(elems, list): - raise TypeError('%s: needs a list of objects' - % self.__class__.__name__) + raise TypeError(f'{self.__class__.__name__}: ' + 'needs a list of objects') if not elems: elems = [] if not dtype and len(elems) == 0: - raise TypeError('%s: specify a data type if list is empty' - % self.__class__.__name__) + raise TypeError(f'{self.__class__.__name__}: ' + 'specify a data type if list is empty') elif not dtype: self.dtype = type(elems[0]) else: self.dtype = dtype if not all([isinstance(e, self.dtype) for e in elems]): - raise TypeError('%s: needs a list of %s' - % (self.__class__.__name__, self.dtype)) + raise TypeError(f'{self.__class__.__name__}: ' + f'needs a list of {self.dtype}') # a bunch of Gnome classes have an id property defined, which we will # prefer @@ -38,7 +39,8 @@ def __init__(self, elems=None, dtype=None): # is hard to reference as a key self._elems = elems[:] - self._d_index = {self._s_id(elem): idx for idx, elem in enumerate(self._elems)} + self._d_index = {self._s_id(elem): idx + for idx, elem in enumerate(self._elems)} self.callbacks = [] @@ -88,20 +90,15 @@ def add(self, elem): try: for e in elem: if not isinstance(e, self.dtype): - raise TypeError('{0}:\n' - 'Expected: {1!r}\n' - 'Got: {2!r}\n of type: {3}'.format(self.__class__.__name__, - self.dtype, - e, - type(e))) + raise TypeError(f'{self.__class__.__name__}:\n' + f'Expected: {self.dtype!r}\n' + f'Got: {e!r}\n of type: {type(e)}') self.add(e) except TypeError: - raise TypeError('{0}:\n' - 'Expected: {1!r} or an iterable of {1}.\n' - 'Got: {2!r}\n of type: {3}'.format(self.__class__.__name__, - self.dtype, - elem, - type(elem))) + raise TypeError(f'{self.__class__.__name__}:\n' + f'Expected: {self.dtype!r} ' + f'or an iterable of {self.dtype}.\n' + f'Got: {elem!r}\n of type: {type(elem)}') def append(self, elem): self.add(elem) @@ -136,10 +133,8 @@ def replace(self, ident, new_elem): raise exception if 'id' is not found. ''' if not isinstance(new_elem, self.dtype): - raise TypeError('{0}: expected {1}, ' - 'got {2}'.format(self.__class__.__name__, - self.dtype, - type(new_elem))) + raise TypeError(f'{self.__class__.__name__}: ' + f'expected {self.dtype}, got {type(new_elem)}') if isinstance(ident, int): l__key = self._s_id(self._elems[ident]) @@ -173,8 +168,7 @@ def index(self, elem): try: idx = self._d_index[idx] except KeyError: - raise ValueError('{0} is not in OrderedCollection' - .format(elem)) + raise ValueError(f'{elem} is not in OrderedCollection') else: idx = self._d_index[elem] except KeyError: @@ -183,8 +177,7 @@ def index(self, elem): try: idx = self._d_index[ident] except KeyError: - raise ValueError('{0} is not in OrderedCollection' - .format(elem)) + raise ValueError(f'{elem} is not in OrderedCollection') return sorted(self._d_index.values()).index(idx) @@ -248,7 +241,7 @@ def __str__(self): itemlist = sorted(list(self._d_index.items()), key=lambda x: x[1]) # reference the value in list - itemlist = [self._elems[v] for (k, v) in itemlist] + itemlist = [self._elems[v] for (_, v) in itemlist] formatter = ' %s,' if len(itemlist) > 12: # should we abbreviate the list at all? @@ -306,26 +299,28 @@ def update(self, cstruct, refs=None): if isinstance(elem, dict): id_ = elem.get('id', None) if id_ is None: - raise ValueError('id of element in OC update dict is missing') + raise ValueError('id of element in OC update dict ' + 'is missing') if id_ in self._d_index: - #obj currently exists in this collection + # obj currently exists in this collection obj = self.get(id_) obj.update(elem) else: - raise ValueError('{0} is not an object contained by this OC'.format(id)) + raise ValueError(f'{id} is not an object contained ' + 'by this OrderedCollection') new_vals.append(obj) elif isinstance(elem, self.dtype): new_vals.append(elem) else: - raise ValueError('update element is not dict or valid type for this OC') + raise ValueError('update element is not dict or valid type ' + 'for this OrderedCollection') self.clear() self.add(new_vals) return self - - #JAH: This is why OCs can be serialized and lists cannot! + # JAH: This is why OCs can be serialized and lists cannot! def to_dict(self): ''' Method takes the instance of ordered collection and outputs a list of @@ -338,7 +333,7 @@ def to_dict(self): for obj in self: try: - obj_type = '{0.__module__}.{0.__class__.__name__}'.format(obj) + obj_type = f'{obj.__module__}.{obj.__class__.__name__}' except AttributeError: obj_type = '{0.__class__.__name__}'.format(obj) item = {'obj_type': obj_type, 'id': self._s_id(obj)} @@ -361,7 +356,7 @@ def register_callback(self, callback, if event not in ('add', 'remove', 'replace'): raise ValueError("Events must be either " "('add', 'remove', 'replace'). " - "{0} is not supported".format(event)) + f"{event} is not supported") self.callbacks.append((callback, events)) diff --git a/py_gnome/gnome/utilities/plume.py b/py_gnome/gnome/utilities/plume.py index 47ff3af9b..a1f0ba9d6 100644 --- a/py_gnome/gnome/utilities/plume.py +++ b/py_gnome/gnome/utilities/plume.py @@ -3,7 +3,6 @@ This module holds classes and supporting code for simulating the vertical plume that is generated by an underwater blowout. """ - from datetime import datetime, timedelta import numpy as np @@ -164,7 +163,7 @@ def __iter__(self): next_time = curr_time + timedelta(seconds=self.time_step_delta) yield (curr_time, list(zip(self.plume.coords, - self.elems_in_range(curr_time, next_time)))) + self.elems_in_range(curr_time, next_time)))) else: step = 0 while True: @@ -174,7 +173,7 @@ def __iter__(self): step += 1 yield (curr_time, list(zip(self.plume.coords, - self.elems_in_range(curr_time, next_time)))) + self.elems_in_range(curr_time, next_time)))) if __name__ == '__main__': diff --git a/py_gnome/gnome/utilities/profiledeco.py b/py_gnome/gnome/utilities/profiledeco.py index c919da3aa..f81829595 100644 --- a/py_gnome/gnome/utilities/profiledeco.py +++ b/py_gnome/gnome/utilities/profiledeco.py @@ -7,12 +7,12 @@ def foo(): sleep(1) -this should profile foo whenever it is called and add it to the global profile stats. +this should profile foo whenever it is called and add it to the +global profile stats. ''' import cProfile import pstats -import io profiler = cProfile.Profile() diff --git a/py_gnome/gnome/utilities/projections.py b/py_gnome/gnome/utilities/projections.py index 9efc75c76..88fcf4be1 100644 --- a/py_gnome/gnome/utilities/projections.py +++ b/py_gnome/gnome/utilities/projections.py @@ -14,13 +14,6 @@ NOTE: all coordinates are takes as (lon, lat, depth) even though depth is always ignored """ - - - - - - - import numpy as np from gnome.gnomeobject import GnomeId from gnome.persist.base_schema import ObjTypeSchema @@ -378,9 +371,9 @@ def meters_to_lonlat(meters, ref_positions): :type ref_positions: NX3, numpy array (Only lat is used here) :returns delta_lon_lat: Differential (delta) positional values - Nx3 numpy array of (delta-lon, delta-lat, delta-z) + Nx3 numpy array of + (delta-lon, delta-lat, delta-z) """ - # make a copy -- don't change meters delta_lon_lat = np.array(meters, dtype=np.float64) if len(delta_lon_lat.shape) == 1: diff --git a/py_gnome/gnome/utilities/rand.py b/py_gnome/gnome/utilities/rand.py index 64e7c9b83..6603ff9ce 100644 --- a/py_gnome/gnome/utilities/rand.py +++ b/py_gnome/gnome/utilities/rand.py @@ -11,13 +11,10 @@ import random -def random_with_persistance( - low, - high, - array=None, # update this array, if provided - persistence=None, - time_step=1., - ): +def random_with_persistance(low, high, + array=None, # update this array, if provided + persistence=None, + time_step=1.): """ Used by gnome to generate a randomness between low and high, which is persistent for duration time_step @@ -45,7 +42,6 @@ def random_with_persistance( should also be the same length array, list or tuple. Give all 3 parameters for each element of the array. """ - # make copies since we don't want to change the original arrays low = np.copy(low) high = np.copy(high) @@ -55,7 +51,7 @@ def random_with_persistance( else: if not isinstance(array, np.ndarray): raise ValueError("If an 'array' is provided for computed values," - " it must be a numpy array") + " it must be a numpy array") # exceptions len_msg = ("Length of 'low', 'high' and 'persistence' arrays" @@ -71,7 +67,8 @@ def random_with_persistance( if np.any(high < low): raise ValueError('The lower bound for random_with_persistance must be ' - 'less than or equal to upper bound for all array elements') + 'less than or equal to upper bound ' + 'for all array elements') if np.all(low == high): array[:] = low[:] @@ -90,8 +87,8 @@ def random_with_persistance( assume any persistence greater than 1 hour is meant to be infinite interface should only allow 15 minute or infinite """ - #u_mask = (persistence > 0) # update mask for values to be changed - u_mask = (persistence > 0) & (persistence <= 900) # update mask for values to be changed + # update mask for values to be changed + u_mask = (persistence > 0) & (persistence <= 900) if np.any(u_mask): if np.any(persistence[u_mask] != abs(time_step)): @@ -104,7 +101,9 @@ def random_with_persistance( readable. """ orig = high[u_mask] - low[u_mask] - l__range = orig * np.sqrt(persistence[u_mask] / float(abs(time_step))) + l__range = orig * np.sqrt( + persistence[u_mask] / float(abs(time_step)) + ) mean = (high[u_mask] + low[u_mask]) / 2. # update the bounds for generating the random number @@ -123,7 +122,6 @@ def seed(seed=1): :param seed: Random number generator should be seeded by this value. Default is 1 """ - cy_helpers.srand(seed) random.seed(seed) np.random.seed(seed) diff --git a/py_gnome/gnome/utilities/save_updater.py b/py_gnome/gnome/utilities/save_updater.py index e26775892..5087ba6d1 100644 --- a/py_gnome/gnome/utilities/save_updater.py +++ b/py_gnome/gnome/utilities/save_updater.py @@ -1,13 +1,15 @@ -# Updates a save loaded using pygnome.Model to the latest version -# !!!!Update the version number in gnome.gnomeobject to be consistent with versioning here - -import json -import logging -import glob -import sys -import contextlib +""" +Updates a save loaded using pygnome.Model to the latest version +!!!!Update the version number in gnome.gnomeobject to be consistent +with versioning here +""" import os +import sys import re +import glob +import logging +import contextlib +import json import zipfile from pathlib import Path @@ -20,14 +22,14 @@ ] NON_WEATHERING_DICT = { - "obj_type": "gnome.spill.substance.NonWeatheringSubstance", - "id": "dummy-id-from-save_updater", - "name": "NonWeatheringSubstance_99", - "initializers": [ - "InitWindages_99.json" - ], - "is_weatherable": False, - "standard_density": 1000.0 + "obj_type": "gnome.spill.substance.NonWeatheringSubstance", + "id": "dummy-id-from-save_updater", + "name": "NonWeatheringSubstance_99", + "initializers": [ + "InitWindages_99.json" + ], + "is_weatherable": False, + "standard_density": 1000.0 } # if needed: @@ -42,6 +44,7 @@ # "name": "InitWindages_99" # } + @contextlib.contextmanager def remember_cwd(new_wd): curdir = os.getcwd() @@ -54,9 +57,10 @@ def remember_cwd(new_wd): def update_savefile(save_directory): - if not isinstance(save_directory, str) or not os.path.isdir(save_directory): - raise ValueError('Must unzip save to directory in order to upgrade it to ' - 'the latest version') + if (not isinstance(save_directory, str) or + not os.path.isdir(save_directory)): + raise ValueError('Must unzip save to directory in order to upgrade it ' + 'to the latest version') messages = [] errors = [] @@ -88,9 +92,9 @@ def update_savefile(save_directory): def v0tov1(messages, errors): ''' - Takes a zipfile containing no version.txt and up-converts it to 'version 1'. - This functions purpose is to upgrade save files to maintain compatibility - after the SpillRefactor upgrades. + Takes a zipfile containing no version.txt and up-converts it + to 'version 1'. This function's purpose is to upgrade save files to + maintain compatibility after the SpillRefactor upgrades. ''' def Substance_from_ElementType(et_json, water): ''' @@ -103,8 +107,9 @@ def Substance_from_ElementType(et_json, water): init['obj_type'] = init['obj_type'].replace('.elements.', '.') if 'substance' not in et_json: ''' - Note the id of the new cstructs. The ID IS required at this stage, because - the load process will use it later to establish references between objects + Note the id of the new cstructs. The ID IS required at this stage, + because the load process will use it later to establish references + between objects ''' substance = NON_WEATHERING_DICT substance["initializers"] = inits @@ -212,9 +217,14 @@ def v1tov2(messages, errors): oils.append(fname) # See if it can be used with the current GnomeOil try: + # I don't know if this is absolutely necessary, + # but we import this locally so as to not have a + # global dependency on any external oil packages. + from gnome.spills.gnome_oil import GnomeOil GnomeOil(**json_) except Exception: - # can't be used with GnomeOIl: replace with NonWeathering + # Can't be used with GnomeOIl: replace with + # NonWeathering log.info(f"Oil: {json_['name']} is not longer valid\n" "You will need re-load an oil, which can be " "obtained from The ADIOS Oil Database:\n" @@ -237,7 +247,8 @@ def v1tov2(messages, errors): if "InitWindages" in init_js["obj_type"]: json_['windage_range'] = init_js['windage_range'] json_['windage_persist'] = init_js['windage_persist'] - json.dump(json_, open(fname, 'w', encoding='utf-8'), indent=4) + json.dump(json_, open(fname, 'w', encoding='utf-8'), + indent=4) files_to_remove.append(wind_init) with open('version.txt', 'w', encoding='utf-8') as vers_file: @@ -249,6 +260,7 @@ def v1tov2(messages, errors): messages.append('**Update from v1 to v2 successful**') return messages, errors + def v2tov3(messages, errors): ''' Takes a zipfile containing version 2 and up-converts it @@ -267,7 +279,7 @@ def v2tov3(messages, errors): spills = [] # things with a "gnome.spill" in the path movers = [] # current_movers, CurrentMover wind_movers = [] # wind_movers, WindMover - srs = [] # SpatialRelease + srs = [] # SpatialRelease for fname in jsonfiles: with open(fname, 'r', encoding='utf-8') as fn: json_ = json.load(fn) @@ -282,12 +294,14 @@ def v2tov3(messages, errors): srs.append((fname, json_)) for fn, sp in spills: - sp['obj_type'] = sp['obj_type'].replace('gnome.spill.', 'gnome.spills.') + sp['obj_type'] = sp['obj_type'].replace('gnome.spill.', + 'gnome.spills.') with open(fn, 'w', encoding='utf-8') as fp: json.dump(sp, fp, indent=4) for fn, sr in srs: - sr['obj_type'] = sr['obj_type'].replace('SpatialRelease', 'PolygonRelease') + sr['obj_type'] = sr['obj_type'].replace('SpatialRelease', + 'PolygonRelease') with open(fn, 'w', encoding='utf-8') as fp: json.dump(sr, fp, indent=4) @@ -330,13 +344,14 @@ def v3tov4(messages, errors): files_to_remove = [] - # search for files which have weathering_data object, added to the files_to_remove list + # search for files which have weathering_data object, added to the + # files_to_remove list for fname in jsonfiles: with open(fname, 'r', encoding='utf-8') as fn: json_ = json.load(fn) if 'obj_type' in json_: - if json_['obj_type'] == "gnome.weatherers.weathering_data.WeatheringData": - files_to_remove.append(fname) + if json_['obj_type'] == "gnome.weatherers.weathering_data.WeatheringData": + files_to_remove.append(fname) # remove weathering_data reference from model file for fname in jsonfiles: @@ -346,7 +361,7 @@ def v3tov4(messages, errors): # this is assuming only one for item in json_['weatherers']: if item in files_to_remove: - json_['weatherers'].remove(item) + json_['weatherers'].remove(item) json.dump(json_, open(fname, 'w', encoding='utf-8'), indent=4) # remove targeted files @@ -356,6 +371,7 @@ def v3tov4(messages, errors): messages.append('**Update from v3 to v4 successful**') return messages, errors + def v4tov5(messages, errors): ''' Takes a zipfile containing version 4 and up-converts it @@ -371,11 +387,10 @@ def v4tov5(messages, errors): jsonfiles = glob.glob('*.json') - # updating the name of spills movers = [] # PyCurrentMover --> CurrentMover - wind_movers = [] #PyWindMover --> WindMover - point_wind_movers = [] #WindMover --> PointWindMover + wind_movers = [] # PyWindMover --> WindMover + point_wind_movers = [] # WindMover --> PointWindMover for fname in jsonfiles: with open(fname, 'r', encoding='utf-8') as fn: json_ = json.load(fn) @@ -398,7 +413,7 @@ def v4tov5(messages, errors): 'WindMover') with open(fn, 'w', encoding='utf-8') as fp: json.dump(mv, fp, indent=4) - + for fn, mv in point_wind_movers: mv['obj_type'] = mv['obj_type'].replace('WindMover', 'PointWindMover') @@ -409,10 +424,10 @@ def v4tov5(messages, errors): vers_file.write('5\n') messages.append('**Update from v4 to v5 successful**') - + return messages, errors - - + + def extract_zipfile(zip_file, to_folder='.'): def work(zf): folders = [name for name in zf.namelist() @@ -441,8 +456,8 @@ def work(zf): log.info('Save file contained invalid names. ' 'Editing extracted json to maintain save file integrity.') for jsonfile in glob.glob(os.path.join(to_folder, '*.json')): - # if any file name edits were made, references may need to be updated too - # otherwise the .json file won't be found + # if any file name edits were made, references may need + # to be updated too otherwise the .json file won't be found contents = None replaced = False with open(jsonfile, 'r', encoding='utf-8') as jf: @@ -469,12 +484,12 @@ def sanitize_filename(fname): NOTE: this looks a lot like html sanitization do we need to remove "<" and ">" from filenames? - and/or should we replace with a placeholder, rather than simply remove? + and/or should we replace with a placeholder, + rather than simply remove? Also -- maybe replace spaces with underscores ... ''' return re.sub(r'[\\\\/*?:"<>|]', "", fname) # return re.sub(r'[\\\\/*?:"<>|]', "", fname).replace(" ", "_") - all_update_steps = [v0tov1, v1tov2, v2tov3, v3tov4, v4tov5] diff --git a/py_gnome/gnome/utilities/schema_decorator.py b/py_gnome/gnome/utilities/schema_decorator.py index 9e9d462a5..541416ce1 100644 --- a/py_gnome/gnome/utilities/schema_decorator.py +++ b/py_gnome/gnome/utilities/schema_decorator.py @@ -6,14 +6,11 @@ for GNOME_ID ? """ - - - - # from future import standard_library # standard_library.install_aliases() # from builtins import * + def serializable(cls): """ make a class serializable @@ -24,7 +21,5 @@ def serializable(cls): print(cls.__name__) type("Schema") Schema(base_schema.ObjTypeSchema) - for attr in cls.__dict__: + for _attr in cls.__dict__: pass - - diff --git a/py_gnome/gnome/utilities/serializable_demo_objects.py b/py_gnome/gnome/utilities/serializable_demo_objects.py index 5e50e931b..483f75e6d 100644 --- a/py_gnome/gnome/utilities/serializable_demo_objects.py +++ b/py_gnome/gnome/utilities/serializable_demo_objects.py @@ -1,9 +1,16 @@ +''' +This file is documentation and a demonstration of how to use Schema objects to +allow Gnome objects to do the following tasks: + Save the object to a zip file + Load the object from a zip file + Get a serialization of the object (JSON) + Deserialize a JSON structure into an instance + Apply a JSON structure as an update +''' +import datetime as dt import numpy as np - import colander -import datetime as dt - from gnome.persist import base_schema from gnome.persist.extend_colander import FilenameSchema @@ -15,15 +22,6 @@ TimeseriesData) from gnome.gnomeobject import GnomeId -''' -This file is documentation and a demonstration of how to use Schema objects to -allow Gnome objects to do the following tasks: - Save the object to a zip file - Load the object from a zip file - Get a serialization of the object (JSON) - Deserialize a JSON structure into an instance - Apply a JSON structure as an update -''' def dates(): return np.array([dt.datetime(2000, 1, 1, 0), @@ -32,11 +30,13 @@ def dates(): dt.datetime(2000, 1, 1, 6), dt.datetime(2000, 1, 1, 8), ]) + def series_data(): - return np.array([1,3,6,10,15]) + return np.array([1, 3, 6, 10, 15]) + def series_data2(): - return np.array([2,6,12,20,30]) + return np.array([2, 6, 12, 20, 30]) class DemoObjSchema(base_schema.ObjTypeSchema): @@ -80,17 +80,19 @@ class DemoObj(GnomeId): _schema = DemoObjSchema - def __init__(self, filename=None, foo_float=None, foo_float_array=None, variable=None, variables=None, **kwargs): + def __init__(self, filename=None, foo_float=None, foo_float_array=None, + variable=None, variables=None, **kwargs): self.filename = filename self.foo_float = 42.0 - self.foo_float_array = [42.0,84.0] + self.foo_float_array = [42.0, 84.0] self.variable = variable self.variables = variables super(DemoObj, self).__init__(**kwargs) @property def timeseries(self): - return [(t, self.variable.variables[0].data[i]) for i, t in enumerate(self.variable.time)] + return [(t, self.variable.variables[0].data[i]) + for i, t in enumerate(self.variable.time)] @classmethod def demo(cls): @@ -102,4 +104,3 @@ def demo(cls): ) return DemoObj(variable=tsv, variables=[tsv, tsv.variables[0]]) - diff --git a/py_gnome/gnome/utilities/shapefile_builder.py b/py_gnome/gnome/utilities/shapefile_builder.py index e90e86918..9d7ae0fdc 100644 --- a/py_gnome/gnome/utilities/shapefile_builder.py +++ b/py_gnome/gnome/utilities/shapefile_builder.py @@ -1,15 +1,12 @@ """Shapefile Builder""" -import geopandas as gpd -import os -import pandas as pd import pathlib -import pyogrio -from shapely.geometry import Point import shutil -import time import warnings -import zipfile + +import geopandas as gpd +import pandas as pd +from shapely.geometry import Point from gnome.utilities.hull import calculate_hull, calculate_contours @@ -17,6 +14,7 @@ warnings.filterwarnings('ignore', message='.*Possibly due to too larger', category=RuntimeWarning, module='pyogrio') + class ShapefileBuilder(object): def __init__(self, filename, zip_output=True, **kwargs): ''' @@ -54,38 +52,47 @@ def write(self): shz_name = self.fullfilename.with_suffix('.shz') # Write out the zipped shapefile full_gdf.to_file(shz_name, driver='ESRI Shapefile', - engine="pyogrio", geometry_type=self.geometry_type) + engine="pyogrio", + geometry_type=self.geometry_type) shutil.move(shz_name, self.fullfilename) else: # Write out the raw shapefile full_gdf.to_file(self.fullfilename, driver='ESRI Shapefile', - engine="pyogrio", geometry_type=self.geometry_type) + engine="pyogrio", + geometry_type=self.geometry_type) return self.fullfilename @property def filename(self): return self.fullfilename + class ParticleShapefileBuilder(ShapefileBuilder): def __init__(self, filename, zip_output=True, **kwargs): ''' :param filename: Full path and basename of the shape file. :param zip: If we should zip the final shapefile results. ''' - super(ParticleShapefileBuilder, self).__init__(filename, zip_output, **kwargs) + super(ParticleShapefileBuilder, self).__init__(filename, zip_output, + **kwargs) def append(self, sc): - """Given a spill container, write out the current particle data to a data frame""" + """ + Given a spill container, write out the current particle data to a + data frame + """ super(ParticleShapefileBuilder, self).append(sc) - frame_data = {'LE_id': sc['id'], - 'Spill_id': sc['spill_num'], - 'Depth': [pos[2] for pos in sc['positions']], - 'Mass': sc['mass'], - 'Age': sc['age'], - 'StatusCode': sc['status_codes'], - 'Time': sc.current_time_stamp.strftime('%Y-%m-%dT%H:%M:%S'), - 'Position': [Point(pt) for pt in sc['positions']] + frame_data = { + 'LE_id': sc['id'], + 'Spill_id': sc['spill_num'], + 'Depth': [pos[2] for pos in sc['positions']], + 'Mass': sc['mass'], + 'Age': sc['age'], + 'StatusCode': sc['status_codes'], + 'Time': sc.current_time_stamp.strftime('%Y-%m-%dT%H:%M:%S'), + 'Position': [Point(pt) for pt in sc['positions']] } + # Some elements are optional... check those here: if 'surface_concentration' in sc and not sc.uncertain: frame_data['Surf_Conc'] = sc['surface_concentration'] @@ -95,7 +102,8 @@ def append(self, sc): frame_data['FracWater'] = sc['frac_water'] if 'density' in sc: frame_data['Density'] = sc['density'] - gdf = gpd.GeoDataFrame(frame_data, crs='epsg:4326', geometry='Position') + gdf = gpd.GeoDataFrame(frame_data, crs='epsg:4326', + geometry='Position') # If we have surface concentration sort by it... if 'surface_concentration' in sc and not sc.uncertain: gdf = gdf.loc[gdf['Surf_Conc'].sort_values().index] @@ -111,33 +119,46 @@ def write(self): self.geometry_type = 'Point' super(ParticleShapefileBuilder, self).write() + def area_in_meters(this_hull): - schema = {'geometry':[this_hull]} - geodataframe = gpd.GeoDataFrame(schema, crs='epsg:4326', geometry='geometry') + schema = {'geometry': [this_hull]} + geodataframe = gpd.GeoDataFrame(schema, crs='epsg:4326', + geometry='geometry') geodataframe_3857 = geodataframe.to_crs(epsg=3857) return geodataframe_3857.area[0] + class BoundaryShapefileBuilder(ShapefileBuilder): def __init__(self, filename, zip_output=True, **kwargs): ''' :param filename: Full path and basename of the shape file. :param zip: If we should zip the final shapefile results. ''' - super(BoundaryShapefileBuilder, self).__init__(filename, zip_output, **kwargs) - - def append(self, sc, separate_by_spill=True, hull_ratio=0.5, hull_allow_holes=False): - """Given a spill container, write out the current boundary data to a data frame""" + super(BoundaryShapefileBuilder, self).__init__(filename, zip_output, + **kwargs) + + def append(self, sc, separate_by_spill=True, hull_ratio=0.5, + hull_allow_holes=False): + """ + Given a spill container, write out the current boundary data to a + data frame + """ super(BoundaryShapefileBuilder, self).append(sc) # Calculate a concave hull - hull = calculate_hull(sc, separate_by_spill=separate_by_spill, ratio=hull_ratio, + hull = calculate_hull(sc, separate_by_spill=separate_by_spill, + ratio=hull_ratio, allow_holes=hull_allow_holes) - # Only process it if we get a hull back. There are cases where the hull is not - # a polygon, and we skip those. + # Only process it if we get a hull back. + # There are cases where the hull is not a polygon, and we skip those. if hull: - frame_data = {'geometry': [hull], - 'area': [area_in_meters(hull)], - 'time': sc.current_time_stamp.strftime('%Y-%m-%dT%H:%M:%S')} - gdf = gpd.GeoDataFrame(frame_data, crs='epsg:4326', geometry='geometry') + frame_data = { + 'geometry': [hull], + 'area': [area_in_meters(hull)], + 'time': sc.current_time_stamp.strftime('%Y-%m-%dT%H:%M:%S') + } + + gdf = gpd.GeoDataFrame(frame_data, crs='epsg:4326', + geometry='geometry') self.data_frames.append(gdf) def write(self): @@ -150,33 +171,39 @@ def write(self): self.geometry_type = 'Polygon' super(BoundaryShapefileBuilder, self).write() + class ContourShapefileBuilder(ShapefileBuilder): def __init__(self, filename, zip_output=True, **kwargs): ''' :param filename: Full path and basename of the shape file. :param zip: If we should zip the final shapefile results. ''' - super(ContourShapefileBuilder, self).__init__(filename, zip_output, **kwargs) - - def append(self, sc, cutoff_struct=None, hull_ratio=0.5, hull_allow_holes=False): - """Given a spill container, write out the contours based on the cutoffs provided""" + super(ContourShapefileBuilder, self).__init__(filename, zip_output, + **kwargs) + + def append(self, sc, cutoff_struct=None, hull_ratio=0.5, + hull_allow_holes=False): + """ + Given a spill container, write out the contours based on the + cutoffs provided + """ # Cutoffs defined in a structure like the following (by spill_num): - #cutoff_struct = {1: {'param': 'mass', - # 'cutoffs': [{'cutoff': 100, - # 'label': 'low'}, - # {'cutoff': 500, - # 'label': 'medium'}, - # {'cutoff': 1000, - # 'label': 'heavy'}]}, - # 2: {'param': 'surf_conc', - # 'cutoffs': [{'cutoff': 0.3, - # 'label': 'low'}, - # {'cutoff': 0.5, - # 'label': 'medium'}, - # {'cutoff': 1.0, - # 'label': 'high'}] - # } - # } + # cutoff_struct = {1: {'param': 'mass', + # 'cutoffs': [{'cutoff': 100, + # 'label': 'low'}, + # {'cutoff': 500, + # 'label': 'medium'}, + # {'cutoff': 1000, + # 'label': 'heavy'}]}, + # 2: {'param': 'surf_conc', + # 'cutoffs': [{'cutoff': 0.3, + # 'label': 'low'}, + # {'cutoff': 0.5, + # 'label': 'medium'}, + # {'cutoff': 1.0, + # 'label': 'high'}] + # } + # } # The cutoffs are ordered array... low to high # First "low" bin is always [data] < cutoffs[current] # Middle bins are cutoffs[previous] < [data] < cutoffs[current] @@ -186,7 +213,8 @@ def append(self, sc, cutoff_struct=None, hull_ratio=0.5, hull_allow_holes=False) # Look in the spill container and get a list of spills # Make sure they are defined in the cutoffs_per_spill # If they are, we loop through each spill - # In each spill we loop through the cutoffs and create subsets of the data + # In each spill we loop through the cutoffs and create subsets + # of the data. # Make a hull around each subset # Create a geodataframe based on the array of generated hulls # Append to the data_frames @@ -196,14 +224,17 @@ def append(self, sc, cutoff_struct=None, hull_ratio=0.5, hull_allow_holes=False) ratio=hull_ratio, allow_holes=hull_allow_holes) if contours: - frame_data = {'geometry': [c['contour'] for c in contours], - 'spill_num': [c['spill_num'] for c in contours], - 'cutoff': [c['cutoff'] for c in contours], - 'cutoff_id': [c['cutoff_id'] for c in contours], - 'color': [c['color'] for c in contours], - 'label': [c['label'] for c in contours], - 'time': sc.current_time_stamp.strftime('%Y-%m-%dT%H:%M:%S')} - gdf = gpd.GeoDataFrame(frame_data, crs='epsg:4326', geometry='geometry') + frame_data = { + 'geometry': [c['contour'] for c in contours], + 'spill_num': [c['spill_num'] for c in contours], + 'cutoff': [c['cutoff'] for c in contours], + 'cutoff_id': [c['cutoff_id'] for c in contours], + 'color': [c['color'] for c in contours], + 'label': [c['label'] for c in contours], + 'time': sc.current_time_stamp.strftime('%Y-%m-%dT%H:%M:%S') + } + gdf = gpd.GeoDataFrame(frame_data, crs='epsg:4326', + geometry='geometry') self.data_frames.append(gdf) def write(self): diff --git a/py_gnome/gnome/utilities/surface_concentration.py b/py_gnome/gnome/utilities/surface_concentration.py index 55ab9137d..1cb0f8ecc 100644 --- a/py_gnome/gnome/utilities/surface_concentration.py +++ b/py_gnome/gnome/utilities/surface_concentration.py @@ -1,16 +1,10 @@ #!/usr/bin/env python - """ Code to compute surface surface_concentration from particles Ultimatley, there may be multiple versions of this -- with Cython optimizationas and all that. """ - - - - - import warnings import numpy as np from scipy.stats import gaussian_kde @@ -30,8 +24,8 @@ def compute_surface_concentration(sc, algorithm): if algorithm == 'kde': surface_conc_kde(sc) else: - raise ValueError('the only surface concentration algorithm currently supported' - 'is "kde"') + raise ValueError('the only surface concentration algorithm ' + 'currently supported is "kde"') def surface_conc_kde(sc): @@ -49,7 +43,7 @@ def surface_conc_kde(sc): spill_num = sc['spill_num'] sc['surface_concentration'] = np.zeros(spill_num.shape[0],) for s in np.unique(spill_num): - sid = np.where(spill_num==s) + sid = np.where(spill_num == s) positions = sc['positions'][sid] mass = sc['mass'][sid] age = sc['age'][sid] @@ -57,19 +51,27 @@ def surface_conc_kde(sc): lon = positions[:, 0] lat = positions[:, 1] - bin_length = 1*3600 #kde will be calculated on particles 0-6hrs, 6-12hrs,... + # kde will be calculated on particles 0-6hrs, 6-12hrs,... + bin_length = 1 * 3600 + t = age.min() max_age = age.max() - while t<=max_age: - id = np.where((age=t)[0] #we only calculate pdf for particles in bin - if len(np.unique(lat_for_kernel))>2 and len(np.unique(lon_for_kernel))>2: # can't compute a kde for less than 3 unique points! + # we only calculate pdf for particles in bin + id_bin = np.where(age_for_kernel >= t)[0] + + # can't compute a kde for less than 3 unique points! + if (len(np.unique(lat_for_kernel)) > 2 and + len(np.unique(lon_for_kernel)) > 2): try: lon0, lat0 = min(lon_for_kernel), min(lat_for_kernel) # FIXME: should use projection code to get this right. @@ -77,15 +79,19 @@ def surface_conc_kde(sc): y = (lat_for_kernel - lat0) * 111325 xy = np.vstack([x, y]) if len(np.unique(mass_for_kernel)) > 1: - kernel = gaussian_kde(xy,weights=mass_for_kernel/mass_for_kernel.sum()) + kernel = gaussian_kde( + xy, + weights=mass_for_kernel / mass_for_kernel.sum() + ) else: kernel = gaussian_kde(xy) if mass_for_kernel.sum() > 0: - c[id[id_bin]] = kernel(xy[:,id_bin]) * mass_for_kernel.sum() + c[id[id_bin]] = kernel(xy[:, id_bin]) * mass_for_kernel.sum() else: - c[id[id_bin]] = kernel(xy[:,id_bin]) * len(mass_for_kernel) + c[id[id_bin]] = kernel(xy[:, id_bin]) * len(mass_for_kernel) except np.linalg.LinAlgError: - warnings.warn("LinAlg error occurred in surface concentration calculations.") + warnings.warn('LinAlg error occurred in ' + 'surface concentration calculations.') t = t + bin_length sc['surface_concentration'][sid] = c diff --git a/py_gnome/gnome/utilities/time_utils.py b/py_gnome/gnome/utilities/time_utils.py index fa608484c..944872b41 100644 --- a/py_gnome/gnome/utilities/time_utils.py +++ b/py_gnome/gnome/utilities/time_utils.py @@ -99,7 +99,6 @@ def date_to_sec(date_times): date_times = [date_times.astype(datetime)] if not isinstance(date_times[0], (datetime, cftime.datetime)): - # if not isinstance(date_times[0], datetime): raise TypeError('date_to_sec only works on datetime and datetime64 ' 'objects. Got a: {}'.format(type(date_times[0]))) diff --git a/py_gnome/gnome/utilities/transforms.py b/py_gnome/gnome/utilities/transforms.py index 13f2a66ab..fdd6a0180 100644 --- a/py_gnome/gnome/utilities/transforms.py +++ b/py_gnome/gnome/utilities/transforms.py @@ -1,6 +1,7 @@ import numpy as np + def r_theta_to_uv_current(r_theta): """ Converts array of current values given with magnitude, direction into @@ -9,7 +10,8 @@ def r_theta_to_uv_current(r_theta): the direction the current moves towards. :param r_theta: NX2 numpy array containing r = r_theta[:,0], - theta = r_theta[:,1]. Theta is in degrees between 0 and 360. + theta = r_theta[:,1]. Theta is in degrees between + 0 and 360. :returns: NX2 numpy array containing the corresponding uv Cartesian velocity vector @@ -21,8 +23,9 @@ def r_theta_to_uv_current(r_theta): def uv_to_r_theta_current(uv): """ - Converts array of current values given with (u,v) current values to magnitude, - direction. Current from 0 deg is (u,v) = (0,1), from 45 deg (u,v) = + Converts array of current values given with (u,v) current values to + magnitude, direction. + Current from 0 deg is (u,v) = (0,1), from 45 deg (u,v) = (1,1) i.e. rotate clockwise from North. In addition, (u,v) represents the direction the current blows towards @@ -38,25 +41,25 @@ def uv_to_r_theta_current(uv): def r_theta_to_uv_wind(r_theta): """ - Converts array of wind values given with magnitude, direction into(u,v) wind - values. Wind from 0 deg is (u,v) = (0,-1), from 45 deg (u,v) = + Converts array of wind values given with magnitude, direction into(u,v) + wind values. Wind from 0 deg is (u,v) = (0,-1), from 45 deg (u,v) = (-1,-1) i.e. rotate clockwise from North. In addition, (u,v) represents the direction the wind blows towards :param r_theta: NX2 numpy array containing r = r_theta[:,0], - theta = r_theta[:,1]. Theta is in degrees between 0 and 360. - :returns: NX2 numpy array containing the corresponding uv Cartesian velocity - vector + theta = r_theta[:,1]. Theta is in degrees between + 0 and 360. + :returns: NX2 numpy array containing the corresponding uv Cartesian + velocity vector """ - r_theta = np.asarray(r_theta, dtype=np.float64).reshape(-1, 2) if np.any(r_theta[:, 1] > 360) or np.any(r_theta[:, 1] < 0): - raise ValueError('input angle in r_theta[:,1] must be between 0 and 360' - ) + raise ValueError('input angle in r_theta[:,1] must be ' + 'between 0 and 360') if np.any(r_theta[:, 0] < 0): - raise ValueError('input magnitude in r_theta[:,0] must be greater than, equal to 0' - ) + raise ValueError('input magnitude in r_theta[:,0] must be ' + 'greater than, equal to 0') rq = np.array(r_theta) rq[:, 1] = np.deg2rad(rq[:, 1]) @@ -88,13 +91,12 @@ def uv_to_r_theta_wind(uv): r_theta = np.zeros_like(uv) r_theta[:, 0] = np.apply_along_axis(np.linalg.norm, 1, uv) - # NOTE: Since desired angle is different from the angle that arctan2 outputs; - # the uv array is transformed (multiply by -1) and atan2 is called with (u,v) - # Only to ensure we get the angle per the Wind convention + # NOTE: Since desired angle is different from the angle that + # arctan2 outputs; the uv array is transformed (multiply by -1) + # and atan2 is called with (u,v) + # Only to ensure we get the angle per the Wind convention uv = -1 * uv # create new uv object - r_theta[:, 1] = (np.rad2deg(np.arctan2(uv[:, 0], uv[:, 1])) + 360) % 360 # 0 to 360 + r_theta[:, 1] = (np.rad2deg(np.arctan2(uv[:, 0], uv[:, 1])) + 360) % 360 return r_theta - - diff --git a/lib_gnome/ADCPMover_c.cpp b/py_gnome/lib_gnome/ADCPMover_c.cpp similarity index 100% rename from lib_gnome/ADCPMover_c.cpp rename to py_gnome/lib_gnome/ADCPMover_c.cpp diff --git a/lib_gnome/ADCPMover_c.h b/py_gnome/lib_gnome/ADCPMover_c.h similarity index 100% rename from lib_gnome/ADCPMover_c.h rename to py_gnome/lib_gnome/ADCPMover_c.h diff --git a/lib_gnome/ADCPTimeValue_c.cpp b/py_gnome/lib_gnome/ADCPTimeValue_c.cpp similarity index 100% rename from lib_gnome/ADCPTimeValue_c.cpp rename to py_gnome/lib_gnome/ADCPTimeValue_c.cpp diff --git a/lib_gnome/ADCPTimeValue_c.h b/py_gnome/lib_gnome/ADCPTimeValue_c.h similarity index 100% rename from lib_gnome/ADCPTimeValue_c.h rename to py_gnome/lib_gnome/ADCPTimeValue_c.h diff --git a/lib_gnome/Basics.h b/py_gnome/lib_gnome/Basics.h similarity index 100% rename from lib_gnome/Basics.h rename to py_gnome/lib_gnome/Basics.h diff --git a/lib_gnome/CATSMover3D_c.cpp b/py_gnome/lib_gnome/CATSMover3D_c.cpp similarity index 100% rename from lib_gnome/CATSMover3D_c.cpp rename to py_gnome/lib_gnome/CATSMover3D_c.cpp diff --git a/lib_gnome/CATSMover3D_c.h b/py_gnome/lib_gnome/CATSMover3D_c.h similarity index 100% rename from lib_gnome/CATSMover3D_c.h rename to py_gnome/lib_gnome/CATSMover3D_c.h diff --git a/lib_gnome/CATSMover_c.cpp b/py_gnome/lib_gnome/CATSMover_c.cpp similarity index 99% rename from lib_gnome/CATSMover_c.cpp rename to py_gnome/lib_gnome/CATSMover_c.cpp index e96aa352c..66bdaba0d 100644 --- a/lib_gnome/CATSMover_c.cpp +++ b/py_gnome/lib_gnome/CATSMover_c.cpp @@ -302,8 +302,10 @@ OSErr CATSMover_c::PrepareForModelStep(const Seconds &model_time, err = this->ComputeVelocityScale(model_time); if (err) + { printError("An error occurred in TCATSMover::PrepareForModelStep"); - + return err; + } // in m/s //note: DIVIDED by timestep because this is later multiplied by the timestep this->fOptimize.isOptimizedForStep = true; diff --git a/lib_gnome/CATSMover_c.h b/py_gnome/lib_gnome/CATSMover_c.h similarity index 100% rename from lib_gnome/CATSMover_c.h rename to py_gnome/lib_gnome/CATSMover_c.h diff --git a/lib_gnome/CMYLIST.CPP b/py_gnome/lib_gnome/CMYLIST.CPP similarity index 100% rename from lib_gnome/CMYLIST.CPP rename to py_gnome/lib_gnome/CMYLIST.CPP diff --git a/lib_gnome/CMYLIST.H b/py_gnome/lib_gnome/CMYLIST.H similarity index 100% rename from lib_gnome/CMYLIST.H rename to py_gnome/lib_gnome/CMYLIST.H diff --git a/lib_gnome/CMap2.cpp b/py_gnome/lib_gnome/CMap2.cpp similarity index 100% rename from lib_gnome/CMap2.cpp rename to py_gnome/lib_gnome/CMap2.cpp diff --git a/lib_gnome/CMap2.h b/py_gnome/lib_gnome/CMap2.h similarity index 100% rename from lib_gnome/CMap2.h rename to py_gnome/lib_gnome/CMap2.h diff --git a/lib_gnome/CMapLayer_c.cpp b/py_gnome/lib_gnome/CMapLayer_c.cpp similarity index 100% rename from lib_gnome/CMapLayer_c.cpp rename to py_gnome/lib_gnome/CMapLayer_c.cpp diff --git a/lib_gnome/ClassID_c.cpp b/py_gnome/lib_gnome/ClassID_c.cpp similarity index 100% rename from lib_gnome/ClassID_c.cpp rename to py_gnome/lib_gnome/ClassID_c.cpp diff --git a/lib_gnome/ClassID_c.h b/py_gnome/lib_gnome/ClassID_c.h similarity index 100% rename from lib_gnome/ClassID_c.h rename to py_gnome/lib_gnome/ClassID_c.h diff --git a/lib_gnome/CompFunctions.cpp b/py_gnome/lib_gnome/CompFunctions.cpp similarity index 100% rename from lib_gnome/CompFunctions.cpp rename to py_gnome/lib_gnome/CompFunctions.cpp diff --git a/lib_gnome/CompFunctions.h b/py_gnome/lib_gnome/CompFunctions.h similarity index 100% rename from lib_gnome/CompFunctions.h rename to py_gnome/lib_gnome/CompFunctions.h diff --git a/lib_gnome/ComponentMover_c.cpp b/py_gnome/lib_gnome/ComponentMover_c.cpp similarity index 100% rename from lib_gnome/ComponentMover_c.cpp rename to py_gnome/lib_gnome/ComponentMover_c.cpp diff --git a/lib_gnome/ComponentMover_c.h b/py_gnome/lib_gnome/ComponentMover_c.h similarity index 100% rename from lib_gnome/ComponentMover_c.h rename to py_gnome/lib_gnome/ComponentMover_c.h diff --git a/lib_gnome/CompoundMap_c.cpp b/py_gnome/lib_gnome/CompoundMap_c.cpp similarity index 100% rename from lib_gnome/CompoundMap_c.cpp rename to py_gnome/lib_gnome/CompoundMap_c.cpp diff --git a/lib_gnome/CompoundMap_c.h b/py_gnome/lib_gnome/CompoundMap_c.h similarity index 100% rename from lib_gnome/CompoundMap_c.h rename to py_gnome/lib_gnome/CompoundMap_c.h diff --git a/lib_gnome/CompoundMover_c.cpp b/py_gnome/lib_gnome/CompoundMover_c.cpp similarity index 100% rename from lib_gnome/CompoundMover_c.cpp rename to py_gnome/lib_gnome/CompoundMover_c.cpp diff --git a/lib_gnome/CompoundMover_c.h b/py_gnome/lib_gnome/CompoundMover_c.h similarity index 100% rename from lib_gnome/CompoundMover_c.h rename to py_gnome/lib_gnome/CompoundMover_c.h diff --git a/lib_gnome/CurrentCycleMover_c.cpp b/py_gnome/lib_gnome/CurrentCycleMover_c.cpp similarity index 100% rename from lib_gnome/CurrentCycleMover_c.cpp rename to py_gnome/lib_gnome/CurrentCycleMover_c.cpp diff --git a/lib_gnome/CurrentCycleMover_c.h b/py_gnome/lib_gnome/CurrentCycleMover_c.h similarity index 100% rename from lib_gnome/CurrentCycleMover_c.h rename to py_gnome/lib_gnome/CurrentCycleMover_c.h diff --git a/lib_gnome/CurrentMover_c.cpp b/py_gnome/lib_gnome/CurrentMover_c.cpp similarity index 100% rename from lib_gnome/CurrentMover_c.cpp rename to py_gnome/lib_gnome/CurrentMover_c.cpp diff --git a/lib_gnome/CurrentMover_c.h b/py_gnome/lib_gnome/CurrentMover_c.h similarity index 100% rename from lib_gnome/CurrentMover_c.h rename to py_gnome/lib_gnome/CurrentMover_c.h diff --git a/lib_gnome/DagStruct.h b/py_gnome/lib_gnome/DagStruct.h similarity index 100% rename from lib_gnome/DagStruct.h rename to py_gnome/lib_gnome/DagStruct.h diff --git a/lib_gnome/DagTree.cpp b/py_gnome/lib_gnome/DagTree.cpp similarity index 100% rename from lib_gnome/DagTree.cpp rename to py_gnome/lib_gnome/DagTree.cpp diff --git a/lib_gnome/DagTree.h b/py_gnome/lib_gnome/DagTree.h similarity index 100% rename from lib_gnome/DagTree.h rename to py_gnome/lib_gnome/DagTree.h diff --git a/lib_gnome/DagTreeIO.cpp b/py_gnome/lib_gnome/DagTreeIO.cpp similarity index 100% rename from lib_gnome/DagTreeIO.cpp rename to py_gnome/lib_gnome/DagTreeIO.cpp diff --git a/lib_gnome/DagTreeIO.h b/py_gnome/lib_gnome/DagTreeIO.h similarity index 100% rename from lib_gnome/DagTreeIO.h rename to py_gnome/lib_gnome/DagTreeIO.h diff --git a/lib_gnome/ExportSymbols.h b/py_gnome/lib_gnome/ExportSymbols.h similarity index 100% rename from lib_gnome/ExportSymbols.h rename to py_gnome/lib_gnome/ExportSymbols.h diff --git a/lib_gnome/GENDEFS.H b/py_gnome/lib_gnome/GENDEFS.H similarity index 100% rename from lib_gnome/GENDEFS.H rename to py_gnome/lib_gnome/GENDEFS.H diff --git a/lib_gnome/GEOMETR2.CPP b/py_gnome/lib_gnome/GEOMETR2.CPP similarity index 100% rename from lib_gnome/GEOMETR2.CPP rename to py_gnome/lib_gnome/GEOMETR2.CPP diff --git a/lib_gnome/GEOMETRY.H b/py_gnome/lib_gnome/GEOMETRY.H similarity index 100% rename from lib_gnome/GEOMETRY.H rename to py_gnome/lib_gnome/GEOMETRY.H diff --git a/lib_gnome/GEOMETRY.cpp b/py_gnome/lib_gnome/GEOMETRY.cpp similarity index 100% rename from lib_gnome/GEOMETRY.cpp rename to py_gnome/lib_gnome/GEOMETRY.cpp diff --git a/lib_gnome/GridCurMover_c.cpp b/py_gnome/lib_gnome/GridCurMover_c.cpp similarity index 100% rename from lib_gnome/GridCurMover_c.cpp rename to py_gnome/lib_gnome/GridCurMover_c.cpp diff --git a/lib_gnome/GridCurMover_c.h b/py_gnome/lib_gnome/GridCurMover_c.h similarity index 100% rename from lib_gnome/GridCurMover_c.h rename to py_gnome/lib_gnome/GridCurMover_c.h diff --git a/lib_gnome/GridCurrentMover_c.cpp b/py_gnome/lib_gnome/GridCurrentMover_c.cpp similarity index 100% rename from lib_gnome/GridCurrentMover_c.cpp rename to py_gnome/lib_gnome/GridCurrentMover_c.cpp diff --git a/lib_gnome/GridCurrentMover_c.h b/py_gnome/lib_gnome/GridCurrentMover_c.h similarity index 100% rename from lib_gnome/GridCurrentMover_c.h rename to py_gnome/lib_gnome/GridCurrentMover_c.h diff --git a/lib_gnome/GridMapUtils.cpp b/py_gnome/lib_gnome/GridMapUtils.cpp similarity index 100% rename from lib_gnome/GridMapUtils.cpp rename to py_gnome/lib_gnome/GridMapUtils.cpp diff --git a/lib_gnome/GridMapUtils.h b/py_gnome/lib_gnome/GridMapUtils.h similarity index 100% rename from lib_gnome/GridMapUtils.h rename to py_gnome/lib_gnome/GridMapUtils.h diff --git a/lib_gnome/GridMap_c.cpp b/py_gnome/lib_gnome/GridMap_c.cpp similarity index 100% rename from lib_gnome/GridMap_c.cpp rename to py_gnome/lib_gnome/GridMap_c.cpp diff --git a/lib_gnome/GridMap_c.h b/py_gnome/lib_gnome/GridMap_c.h similarity index 100% rename from lib_gnome/GridMap_c.h rename to py_gnome/lib_gnome/GridMap_c.h diff --git a/lib_gnome/GridVel_c.h b/py_gnome/lib_gnome/GridVel_c.h similarity index 100% rename from lib_gnome/GridVel_c.h rename to py_gnome/lib_gnome/GridVel_c.h diff --git a/lib_gnome/GridWindMover_c.cpp b/py_gnome/lib_gnome/GridWindMover_c.cpp similarity index 100% rename from lib_gnome/GridWindMover_c.cpp rename to py_gnome/lib_gnome/GridWindMover_c.cpp diff --git a/lib_gnome/GridWindMover_c.h b/py_gnome/lib_gnome/GridWindMover_c.h similarity index 100% rename from lib_gnome/GridWindMover_c.h rename to py_gnome/lib_gnome/GridWindMover_c.h diff --git a/lib_gnome/GridWndMover_c.cpp b/py_gnome/lib_gnome/GridWndMover_c.cpp similarity index 100% rename from lib_gnome/GridWndMover_c.cpp rename to py_gnome/lib_gnome/GridWndMover_c.cpp diff --git a/lib_gnome/GridWndMover_c.h b/py_gnome/lib_gnome/GridWndMover_c.h similarity index 100% rename from lib_gnome/GridWndMover_c.h rename to py_gnome/lib_gnome/GridWndMover_c.h diff --git a/lib_gnome/IceMover_c.cpp b/py_gnome/lib_gnome/IceMover_c.cpp similarity index 100% rename from lib_gnome/IceMover_c.cpp rename to py_gnome/lib_gnome/IceMover_c.cpp diff --git a/lib_gnome/IceMover_c.h b/py_gnome/lib_gnome/IceMover_c.h similarity index 100% rename from lib_gnome/IceMover_c.h rename to py_gnome/lib_gnome/IceMover_c.h diff --git a/lib_gnome/IceWindMover_c.cpp b/py_gnome/lib_gnome/IceWindMover_c.cpp similarity index 100% rename from lib_gnome/IceWindMover_c.cpp rename to py_gnome/lib_gnome/IceWindMover_c.cpp diff --git a/lib_gnome/IceWindMover_c.h b/py_gnome/lib_gnome/IceWindMover_c.h similarity index 100% rename from lib_gnome/IceWindMover_c.h rename to py_gnome/lib_gnome/IceWindMover_c.h diff --git a/lib_gnome/LEList_c.cpp b/py_gnome/lib_gnome/LEList_c.cpp similarity index 100% rename from lib_gnome/LEList_c.cpp rename to py_gnome/lib_gnome/LEList_c.cpp diff --git a/lib_gnome/LEList_c.h b/py_gnome/lib_gnome/LEList_c.h similarity index 100% rename from lib_gnome/LEList_c.h rename to py_gnome/lib_gnome/LEList_c.h diff --git a/lib_gnome/LocaleWizard_c.cpp b/py_gnome/lib_gnome/LocaleWizard_c.cpp similarity index 100% rename from lib_gnome/LocaleWizard_c.cpp rename to py_gnome/lib_gnome/LocaleWizard_c.cpp diff --git a/lib_gnome/LocaleWizard_c.h b/py_gnome/lib_gnome/LocaleWizard_c.h similarity index 100% rename from lib_gnome/LocaleWizard_c.h rename to py_gnome/lib_gnome/LocaleWizard_c.h diff --git a/lib_gnome/MYRANDOM.CPP b/py_gnome/lib_gnome/MYRANDOM.CPP similarity index 100% rename from lib_gnome/MYRANDOM.CPP rename to py_gnome/lib_gnome/MYRANDOM.CPP diff --git a/lib_gnome/MYRANDOM.H b/py_gnome/lib_gnome/MYRANDOM.H similarity index 100% rename from lib_gnome/MYRANDOM.H rename to py_gnome/lib_gnome/MYRANDOM.H diff --git a/lib_gnome/MakeDagTree.cpp b/py_gnome/lib_gnome/MakeDagTree.cpp similarity index 100% rename from lib_gnome/MakeDagTree.cpp rename to py_gnome/lib_gnome/MakeDagTree.cpp diff --git a/lib_gnome/MakeTriangles.cpp b/py_gnome/lib_gnome/MakeTriangles.cpp similarity index 100% rename from lib_gnome/MakeTriangles.cpp rename to py_gnome/lib_gnome/MakeTriangles.cpp diff --git a/lib_gnome/Map3D_c.cpp b/py_gnome/lib_gnome/Map3D_c.cpp similarity index 100% rename from lib_gnome/Map3D_c.cpp rename to py_gnome/lib_gnome/Map3D_c.cpp diff --git a/lib_gnome/Map3D_c.h b/py_gnome/lib_gnome/Map3D_c.h similarity index 100% rename from lib_gnome/Map3D_c.h rename to py_gnome/lib_gnome/Map3D_c.h diff --git a/lib_gnome/MapUtils.cpp b/py_gnome/lib_gnome/MapUtils.cpp similarity index 100% rename from lib_gnome/MapUtils.cpp rename to py_gnome/lib_gnome/MapUtils.cpp diff --git a/lib_gnome/MapUtils.h b/py_gnome/lib_gnome/MapUtils.h similarity index 100% rename from lib_gnome/MapUtils.h rename to py_gnome/lib_gnome/MapUtils.h diff --git a/lib_gnome/Map_c.cpp b/py_gnome/lib_gnome/Map_c.cpp similarity index 100% rename from lib_gnome/Map_c.cpp rename to py_gnome/lib_gnome/Map_c.cpp diff --git a/lib_gnome/Map_c.h b/py_gnome/lib_gnome/Map_c.h similarity index 100% rename from lib_gnome/Map_c.h rename to py_gnome/lib_gnome/Map_c.h diff --git a/lib_gnome/MemUtils.cpp b/py_gnome/lib_gnome/MemUtils.cpp similarity index 100% rename from lib_gnome/MemUtils.cpp rename to py_gnome/lib_gnome/MemUtils.cpp diff --git a/lib_gnome/MemUtils.h b/py_gnome/lib_gnome/MemUtils.h similarity index 100% rename from lib_gnome/MemUtils.h rename to py_gnome/lib_gnome/MemUtils.h diff --git a/lib_gnome/Model_c.cpp b/py_gnome/lib_gnome/Model_c.cpp similarity index 100% rename from lib_gnome/Model_c.cpp rename to py_gnome/lib_gnome/Model_c.cpp diff --git a/lib_gnome/Model_c.h b/py_gnome/lib_gnome/Model_c.h similarity index 100% rename from lib_gnome/Model_c.h rename to py_gnome/lib_gnome/Model_c.h diff --git a/lib_gnome/Mover_c.cpp b/py_gnome/lib_gnome/Mover_c.cpp similarity index 100% rename from lib_gnome/Mover_c.cpp rename to py_gnome/lib_gnome/Mover_c.cpp diff --git a/lib_gnome/Mover_c.h b/py_gnome/lib_gnome/Mover_c.h similarity index 100% rename from lib_gnome/Mover_c.h rename to py_gnome/lib_gnome/Mover_c.h diff --git a/lib_gnome/NetCDFMoverCurv_c.cpp b/py_gnome/lib_gnome/NetCDFMoverCurv_c.cpp similarity index 100% rename from lib_gnome/NetCDFMoverCurv_c.cpp rename to py_gnome/lib_gnome/NetCDFMoverCurv_c.cpp diff --git a/lib_gnome/NetCDFMoverCurv_c.h b/py_gnome/lib_gnome/NetCDFMoverCurv_c.h similarity index 100% rename from lib_gnome/NetCDFMoverCurv_c.h rename to py_gnome/lib_gnome/NetCDFMoverCurv_c.h diff --git a/lib_gnome/NetCDFMoverTri_c.cpp b/py_gnome/lib_gnome/NetCDFMoverTri_c.cpp similarity index 100% rename from lib_gnome/NetCDFMoverTri_c.cpp rename to py_gnome/lib_gnome/NetCDFMoverTri_c.cpp diff --git a/lib_gnome/NetCDFMoverTri_c.h b/py_gnome/lib_gnome/NetCDFMoverTri_c.h similarity index 100% rename from lib_gnome/NetCDFMoverTri_c.h rename to py_gnome/lib_gnome/NetCDFMoverTri_c.h diff --git a/lib_gnome/NetCDFMover_c.cpp b/py_gnome/lib_gnome/NetCDFMover_c.cpp similarity index 100% rename from lib_gnome/NetCDFMover_c.cpp rename to py_gnome/lib_gnome/NetCDFMover_c.cpp diff --git a/lib_gnome/NetCDFMover_c.h b/py_gnome/lib_gnome/NetCDFMover_c.h similarity index 100% rename from lib_gnome/NetCDFMover_c.h rename to py_gnome/lib_gnome/NetCDFMover_c.h diff --git a/lib_gnome/NetCDFWindMoverCurv_c.cpp b/py_gnome/lib_gnome/NetCDFWindMoverCurv_c.cpp similarity index 100% rename from lib_gnome/NetCDFWindMoverCurv_c.cpp rename to py_gnome/lib_gnome/NetCDFWindMoverCurv_c.cpp diff --git a/lib_gnome/NetCDFWindMoverCurv_c.h b/py_gnome/lib_gnome/NetCDFWindMoverCurv_c.h similarity index 100% rename from lib_gnome/NetCDFWindMoverCurv_c.h rename to py_gnome/lib_gnome/NetCDFWindMoverCurv_c.h diff --git a/lib_gnome/NetCDFWindMover_c.cpp b/py_gnome/lib_gnome/NetCDFWindMover_c.cpp similarity index 100% rename from lib_gnome/NetCDFWindMover_c.cpp rename to py_gnome/lib_gnome/NetCDFWindMover_c.cpp diff --git a/lib_gnome/NetCDFWindMover_c.h b/py_gnome/lib_gnome/NetCDFWindMover_c.h similarity index 100% rename from lib_gnome/NetCDFWindMover_c.h rename to py_gnome/lib_gnome/NetCDFWindMover_c.h diff --git a/lib_gnome/OLEList_c.cpp b/py_gnome/lib_gnome/OLEList_c.cpp similarity index 100% rename from lib_gnome/OLEList_c.cpp rename to py_gnome/lib_gnome/OLEList_c.cpp diff --git a/lib_gnome/OLEList_c.h b/py_gnome/lib_gnome/OLEList_c.h similarity index 100% rename from lib_gnome/OLEList_c.h rename to py_gnome/lib_gnome/OLEList_c.h diff --git a/lib_gnome/OSSMTimeValue_c.cpp b/py_gnome/lib_gnome/OSSMTimeValue_c.cpp similarity index 100% rename from lib_gnome/OSSMTimeValue_c.cpp rename to py_gnome/lib_gnome/OSSMTimeValue_c.cpp diff --git a/lib_gnome/OSSMTimeValue_c.h b/py_gnome/lib_gnome/OSSMTimeValue_c.h similarity index 100% rename from lib_gnome/OSSMTimeValue_c.h rename to py_gnome/lib_gnome/OSSMTimeValue_c.h diff --git a/lib_gnome/OSSMWeatherer_c.cpp b/py_gnome/lib_gnome/OSSMWeatherer_c.cpp similarity index 100% rename from lib_gnome/OSSMWeatherer_c.cpp rename to py_gnome/lib_gnome/OSSMWeatherer_c.cpp diff --git a/lib_gnome/OSSMWeatherer_c.h b/py_gnome/lib_gnome/OSSMWeatherer_c.h similarity index 100% rename from lib_gnome/OSSMWeatherer_c.h rename to py_gnome/lib_gnome/OSSMWeatherer_c.h diff --git a/lib_gnome/OUTILS.H b/py_gnome/lib_gnome/OUTILS.H similarity index 100% rename from lib_gnome/OUTILS.H rename to py_gnome/lib_gnome/OUTILS.H diff --git a/lib_gnome/OUTILS.cpp b/py_gnome/lib_gnome/OUTILS.cpp similarity index 100% rename from lib_gnome/OUTILS.cpp rename to py_gnome/lib_gnome/OUTILS.cpp diff --git a/lib_gnome/ObjectUtils.h b/py_gnome/lib_gnome/ObjectUtils.h similarity index 100% rename from lib_gnome/ObjectUtils.h rename to py_gnome/lib_gnome/ObjectUtils.h diff --git a/lib_gnome/PtCurMap_c.cpp b/py_gnome/lib_gnome/PtCurMap_c.cpp similarity index 100% rename from lib_gnome/PtCurMap_c.cpp rename to py_gnome/lib_gnome/PtCurMap_c.cpp diff --git a/lib_gnome/PtCurMap_c.h b/py_gnome/lib_gnome/PtCurMap_c.h similarity index 100% rename from lib_gnome/PtCurMap_c.h rename to py_gnome/lib_gnome/PtCurMap_c.h diff --git a/lib_gnome/PtCurMover_c.cpp b/py_gnome/lib_gnome/PtCurMover_c.cpp similarity index 100% rename from lib_gnome/PtCurMover_c.cpp rename to py_gnome/lib_gnome/PtCurMover_c.cpp diff --git a/lib_gnome/PtCurMover_c.h b/py_gnome/lib_gnome/PtCurMover_c.h similarity index 100% rename from lib_gnome/PtCurMover_c.h rename to py_gnome/lib_gnome/PtCurMover_c.h diff --git a/lib_gnome/Random3D_c.cpp b/py_gnome/lib_gnome/Random3D_c.cpp similarity index 100% rename from lib_gnome/Random3D_c.cpp rename to py_gnome/lib_gnome/Random3D_c.cpp diff --git a/lib_gnome/Random3D_c.h b/py_gnome/lib_gnome/Random3D_c.h similarity index 100% rename from lib_gnome/Random3D_c.h rename to py_gnome/lib_gnome/Random3D_c.h diff --git a/lib_gnome/RandomVertical_c.cpp b/py_gnome/lib_gnome/RandomVertical_c.cpp similarity index 100% rename from lib_gnome/RandomVertical_c.cpp rename to py_gnome/lib_gnome/RandomVertical_c.cpp diff --git a/lib_gnome/RandomVertical_c.h b/py_gnome/lib_gnome/RandomVertical_c.h similarity index 100% rename from lib_gnome/RandomVertical_c.h rename to py_gnome/lib_gnome/RandomVertical_c.h diff --git a/lib_gnome/Random_c.cpp b/py_gnome/lib_gnome/Random_c.cpp similarity index 100% rename from lib_gnome/Random_c.cpp rename to py_gnome/lib_gnome/Random_c.cpp diff --git a/lib_gnome/Random_c.h b/py_gnome/lib_gnome/Random_c.h similarity index 100% rename from lib_gnome/Random_c.h rename to py_gnome/lib_gnome/Random_c.h diff --git a/lib_gnome/RectGridVeL_c.cpp b/py_gnome/lib_gnome/RectGridVeL_c.cpp similarity index 100% rename from lib_gnome/RectGridVeL_c.cpp rename to py_gnome/lib_gnome/RectGridVeL_c.cpp diff --git a/lib_gnome/RectGridVeL_c.h b/py_gnome/lib_gnome/RectGridVeL_c.h similarity index 100% rename from lib_gnome/RectGridVeL_c.h rename to py_gnome/lib_gnome/RectGridVeL_c.h diff --git a/lib_gnome/RectUtils.cpp b/py_gnome/lib_gnome/RectUtils.cpp similarity index 100% rename from lib_gnome/RectUtils.cpp rename to py_gnome/lib_gnome/RectUtils.cpp diff --git a/lib_gnome/RectUtils.h b/py_gnome/lib_gnome/RectUtils.h similarity index 100% rename from lib_gnome/RectUtils.h rename to py_gnome/lib_gnome/RectUtils.h diff --git a/lib_gnome/Replacements.cpp b/py_gnome/lib_gnome/Replacements.cpp similarity index 100% rename from lib_gnome/Replacements.cpp rename to py_gnome/lib_gnome/Replacements.cpp diff --git a/lib_gnome/Replacements.h b/py_gnome/lib_gnome/Replacements.h similarity index 100% rename from lib_gnome/Replacements.h rename to py_gnome/lib_gnome/Replacements.h diff --git a/lib_gnome/RiseVelocity_c.cpp b/py_gnome/lib_gnome/RiseVelocity_c.cpp similarity index 100% rename from lib_gnome/RiseVelocity_c.cpp rename to py_gnome/lib_gnome/RiseVelocity_c.cpp diff --git a/lib_gnome/RiseVelocity_c.h b/py_gnome/lib_gnome/RiseVelocity_c.h similarity index 100% rename from lib_gnome/RiseVelocity_c.h rename to py_gnome/lib_gnome/RiseVelocity_c.h diff --git a/lib_gnome/Shio.h b/py_gnome/lib_gnome/Shio.h similarity index 100% rename from lib_gnome/Shio.h rename to py_gnome/lib_gnome/Shio.h diff --git a/lib_gnome/ShioCurrent1.cpp b/py_gnome/lib_gnome/ShioCurrent1.cpp similarity index 100% rename from lib_gnome/ShioCurrent1.cpp rename to py_gnome/lib_gnome/ShioCurrent1.cpp diff --git a/lib_gnome/ShioCurrent2.cpp b/py_gnome/lib_gnome/ShioCurrent2.cpp similarity index 100% rename from lib_gnome/ShioCurrent2.cpp rename to py_gnome/lib_gnome/ShioCurrent2.cpp diff --git a/lib_gnome/ShioHeight.cpp b/py_gnome/lib_gnome/ShioHeight.cpp similarity index 100% rename from lib_gnome/ShioHeight.cpp rename to py_gnome/lib_gnome/ShioHeight.cpp diff --git a/lib_gnome/ShioTimeValue_c.cpp b/py_gnome/lib_gnome/ShioTimeValue_c.cpp similarity index 100% rename from lib_gnome/ShioTimeValue_c.cpp rename to py_gnome/lib_gnome/ShioTimeValue_c.cpp diff --git a/lib_gnome/ShioTimeValue_c.h b/py_gnome/lib_gnome/ShioTimeValue_c.h similarity index 100% rename from lib_gnome/ShioTimeValue_c.h rename to py_gnome/lib_gnome/ShioTimeValue_c.h diff --git a/lib_gnome/StringFunctions.cpp b/py_gnome/lib_gnome/StringFunctions.cpp similarity index 100% rename from lib_gnome/StringFunctions.cpp rename to py_gnome/lib_gnome/StringFunctions.cpp diff --git a/lib_gnome/StringFunctions.h b/py_gnome/lib_gnome/StringFunctions.h similarity index 100% rename from lib_gnome/StringFunctions.h rename to py_gnome/lib_gnome/StringFunctions.h diff --git a/lib_gnome/TideCurCycleMover_c.cpp b/py_gnome/lib_gnome/TideCurCycleMover_c.cpp similarity index 100% rename from lib_gnome/TideCurCycleMover_c.cpp rename to py_gnome/lib_gnome/TideCurCycleMover_c.cpp diff --git a/lib_gnome/TideCurCycleMover_c.h b/py_gnome/lib_gnome/TideCurCycleMover_c.h similarity index 100% rename from lib_gnome/TideCurCycleMover_c.h rename to py_gnome/lib_gnome/TideCurCycleMover_c.h diff --git a/lib_gnome/TimeGridVel_c.cpp b/py_gnome/lib_gnome/TimeGridVel_c.cpp similarity index 100% rename from lib_gnome/TimeGridVel_c.cpp rename to py_gnome/lib_gnome/TimeGridVel_c.cpp diff --git a/lib_gnome/TimeGridVel_c.h b/py_gnome/lib_gnome/TimeGridVel_c.h similarity index 100% rename from lib_gnome/TimeGridVel_c.h rename to py_gnome/lib_gnome/TimeGridVel_c.h diff --git a/lib_gnome/TimeGridWind_c.cpp b/py_gnome/lib_gnome/TimeGridWind_c.cpp similarity index 100% rename from lib_gnome/TimeGridWind_c.cpp rename to py_gnome/lib_gnome/TimeGridWind_c.cpp diff --git a/lib_gnome/TimeGridWind_c.h b/py_gnome/lib_gnome/TimeGridWind_c.h similarity index 100% rename from lib_gnome/TimeGridWind_c.h rename to py_gnome/lib_gnome/TimeGridWind_c.h diff --git a/lib_gnome/TimeValue_c.cpp b/py_gnome/lib_gnome/TimeValue_c.cpp similarity index 100% rename from lib_gnome/TimeValue_c.cpp rename to py_gnome/lib_gnome/TimeValue_c.cpp diff --git a/lib_gnome/TimeValue_c.h b/py_gnome/lib_gnome/TimeValue_c.h similarity index 100% rename from lib_gnome/TimeValue_c.h rename to py_gnome/lib_gnome/TimeValue_c.h diff --git a/lib_gnome/TimeValuesIO.cpp b/py_gnome/lib_gnome/TimeValuesIO.cpp similarity index 100% rename from lib_gnome/TimeValuesIO.cpp rename to py_gnome/lib_gnome/TimeValuesIO.cpp diff --git a/lib_gnome/TimeValuesIO.h b/py_gnome/lib_gnome/TimeValuesIO.h similarity index 100% rename from lib_gnome/TimeValuesIO.h rename to py_gnome/lib_gnome/TimeValuesIO.h diff --git a/lib_gnome/TriCurMover_c.cpp b/py_gnome/lib_gnome/TriCurMover_c.cpp similarity index 100% rename from lib_gnome/TriCurMover_c.cpp rename to py_gnome/lib_gnome/TriCurMover_c.cpp diff --git a/lib_gnome/TriCurMover_c.h b/py_gnome/lib_gnome/TriCurMover_c.h similarity index 100% rename from lib_gnome/TriCurMover_c.h rename to py_gnome/lib_gnome/TriCurMover_c.h diff --git a/lib_gnome/TriGridVel3D_c.cpp b/py_gnome/lib_gnome/TriGridVel3D_c.cpp similarity index 100% rename from lib_gnome/TriGridVel3D_c.cpp rename to py_gnome/lib_gnome/TriGridVel3D_c.cpp diff --git a/lib_gnome/TriGridVel3D_c.h b/py_gnome/lib_gnome/TriGridVel3D_c.h similarity index 100% rename from lib_gnome/TriGridVel3D_c.h rename to py_gnome/lib_gnome/TriGridVel3D_c.h diff --git a/lib_gnome/TriGridVel_c.cpp b/py_gnome/lib_gnome/TriGridVel_c.cpp similarity index 100% rename from lib_gnome/TriGridVel_c.cpp rename to py_gnome/lib_gnome/TriGridVel_c.cpp diff --git a/lib_gnome/TriGridVel_c.h b/py_gnome/lib_gnome/TriGridVel_c.h similarity index 100% rename from lib_gnome/TriGridVel_c.h rename to py_gnome/lib_gnome/TriGridVel_c.h diff --git a/lib_gnome/TypeDefs.h b/py_gnome/lib_gnome/TypeDefs.h similarity index 100% rename from lib_gnome/TypeDefs.h rename to py_gnome/lib_gnome/TypeDefs.h diff --git a/lib_gnome/Units.h b/py_gnome/lib_gnome/Units.h similarity index 100% rename from lib_gnome/Units.h rename to py_gnome/lib_gnome/Units.h diff --git a/lib_gnome/VectMap_c.cpp b/py_gnome/lib_gnome/VectMap_c.cpp similarity index 100% rename from lib_gnome/VectMap_c.cpp rename to py_gnome/lib_gnome/VectMap_c.cpp diff --git a/lib_gnome/VectMap_c.h b/py_gnome/lib_gnome/VectMap_c.h similarity index 100% rename from lib_gnome/VectMap_c.h rename to py_gnome/lib_gnome/VectMap_c.h diff --git a/lib_gnome/Weatherer_c.cpp b/py_gnome/lib_gnome/Weatherer_c.cpp similarity index 100% rename from lib_gnome/Weatherer_c.cpp rename to py_gnome/lib_gnome/Weatherer_c.cpp diff --git a/lib_gnome/Weatherer_c.h b/py_gnome/lib_gnome/Weatherer_c.h similarity index 100% rename from lib_gnome/Weatherer_c.h rename to py_gnome/lib_gnome/Weatherer_c.h diff --git a/lib_gnome/Weatherers_c.cpp b/py_gnome/lib_gnome/Weatherers_c.cpp similarity index 100% rename from lib_gnome/Weatherers_c.cpp rename to py_gnome/lib_gnome/Weatherers_c.cpp diff --git a/lib_gnome/Weatherers_c.h b/py_gnome/lib_gnome/Weatherers_c.h similarity index 100% rename from lib_gnome/Weatherers_c.h rename to py_gnome/lib_gnome/Weatherers_c.h diff --git a/lib_gnome/WindMover_c.cpp b/py_gnome/lib_gnome/WindMover_c.cpp similarity index 100% rename from lib_gnome/WindMover_c.cpp rename to py_gnome/lib_gnome/WindMover_c.cpp diff --git a/lib_gnome/WindMover_c.h b/py_gnome/lib_gnome/WindMover_c.h similarity index 100% rename from lib_gnome/WindMover_c.h rename to py_gnome/lib_gnome/WindMover_c.h diff --git a/lib_gnome/my_build_list.h b/py_gnome/lib_gnome/my_build_list.h similarity index 100% rename from lib_gnome/my_build_list.h rename to py_gnome/lib_gnome/my_build_list.h diff --git a/py_gnome/pyproject.toml b/py_gnome/pyproject.toml new file mode 100644 index 000000000..c48d4366b --- /dev/null +++ b/py_gnome/pyproject.toml @@ -0,0 +1,58 @@ + +[build-system] +# note: using setuptools_scm for file finding, not versioning +# we should probably do versioning later, but it requires tagging the git repo properly +requires = ["scikit-build-core", "cython", "numpy"] +build-backend = "scikit_build_core.build" + +[project] +name = "gnome" +authors = [ + {name = "Gnome team at NOAA/ORR/ERD", email = "orr.gnome@noaa.gov"}, +] +description = "Particle tracking code with features specifically for oil spill modeling." +requires-python = ">=3.9" +keywords = ["oilspill", "modeling", "particle-tracking"] +license = {text = "CC0-1.0"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: C++", + "Programming Language :: Cython", + "Intended Audience :: Science/Research", + "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", + "Natural Language :: English", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Oceanography", +] + +version = "1.1.7dev" +readme = "README.rst" + +# dynamic = ["version", "readme"] + +# really hard to do with conda compatibility :-) +# dependencies = [ +# "requests", +# 'importlib-metadata; python_version<"3.8"', +# ] + +[tool.setuptools_scm] +root = ".." +# write_to = "py_gnome/gnome/_version.py" + +[tool.setuptools] +zip-safe = false + +[tool.setuptools.packages.find] +where = ["gnome"] # list of folders that contain the packages (["."] by default) +include = ["gnome*"] # package names should match these glob patterns (["*"] by default) +# exclude = ["my_package.tests*"] # exclude packages matching these glob patterns (empty by default) +namespaces = false # to disable scanning PEP 420 namespaces (true by default) + +[tool.scikit-build] +wheel.py-api = "py3" + +# [project.optional-dependencies] diff --git a/py_gnome/scripts/example_scripts/current_uncertainty/current_uncertainty_example.py b/py_gnome/scripts/example_scripts/current_uncertainty/current_uncertainty_example.py index 61961a503..6da3d55f0 100644 --- a/py_gnome/scripts/example_scripts/current_uncertainty/current_uncertainty_example.py +++ b/py_gnome/scripts/example_scripts/current_uncertainty/current_uncertainty_example.py @@ -49,7 +49,7 @@ # ) model.movers += current_mover -# Turn Diffusion off to see the effects of the uncertainty +# Turn Diffusion down to see the effects of the uncertainty model.movers += gs.RandomMover(diffusion_coef=1e1) renderer = gs.Renderer(mymap, diff --git a/py_gnome/scripts/testing_scripts/script_3dcurrents_hawaii/script_plume_HI.py b/py_gnome/scripts/testing_scripts/script_3dcurrents_hawaii/script_plume_HI.py index 383527f43..d51f81369 100644 --- a/py_gnome/scripts/testing_scripts/script_3dcurrents_hawaii/script_plume_HI.py +++ b/py_gnome/scripts/testing_scripts/script_3dcurrents_hawaii/script_plume_HI.py @@ -84,6 +84,11 @@ def make_model(): horizontal_diffusion_coef_below_ml = 100000, mixed_layer_depth=20) + ncout = gs.NetCDFOutput(filename="HI_output.nc", + output_timestep=gs.hours(1), + ) + + model.outputters += ncout return model diff --git a/py_gnome/setup.py b/py_gnome/setup.py index 6b52cdab4..c47ec3183 100755 --- a/py_gnome/setup.py +++ b/py_gnome/setup.py @@ -12,42 +12,44 @@ (which happens in the gnome.__init__.py file) """ -import os -import sys -import sysconfig +import datetime import importlib import glob +import os +from pathlib import Path +import platform import shutil -import datetime +import sys +import sysconfig + # to support "develop" mode: import setuptools -from setuptools import setup, find_packages +from setuptools import setup + from distutils.command.clean import clean from distutils.extension import Extension from Cython.Distutils import build_ext -import platform from git import Repo import numpy as np -# could run setup from anywhere +# could run setup from anywhere (though not really??) SETUP_PATH = os.path.dirname(os.path.abspath(__file__)) # the extension used for compiled modules -# # why is this so ugly?!? on py2? +# does this not work ?? # comp_modules_ext = sysconfig.get_config_var('EXT_SUFFIX') # if comp_modules_ext is None: # comp_modules_ext = '.lib' if 'win' in sys.platform else ".so" -if sys.version_info.major == 3: +if sys.version_info.major < 3 or sys.version_info.minor < 9: + raise NotImplementedError("PyGNOME can only be built with Python 3.9 or above") +else: py_impl = ['.'] + [c.lower() for c in platform.python_implementation() if c.isupper()] py_impl = py_impl + sysconfig.get_python_version().split('.') + ['-'] + [sysconfig.get_platform().replace('-','_')] win_comp_modules_ext = ''.join(py_impl) + '.lib' -else: - win_comp_modules_ext = '.lib' - # cd to SETUP_PATH, run develop or install, then cd back CWD = os.getcwd() @@ -57,8 +59,6 @@ repo = Repo('../.') branch_name = repo.active_branch.name last_update = next(repo.iter_commits()).committed_datetime.isoformat(), - # except TypeError: - # branch_name = 'no-branch' except: # anything goes wrong -- we want to keep moving branch_name = 'no-branch' last_update = datetime.datetime.now().isoformat() @@ -182,8 +182,6 @@ def get_netcdf_libs(): include_dir = include_dir.decode("ASCII").strip() lib_dir = lib_dir.decode("ASCII") libs = [l.decode("ASCII") for l in libs] - # print("found netcdf install:") - # print([lib_dir, libs, include_dir]) return lib_dir, libs, include_dir except OSError: raise NotImplementedError("this setup.py needs nc-config " @@ -204,10 +202,8 @@ def get_conda_includes_win(): # maybe this will work with Windows, too" at least with conda? if sys.platform.startswith("linux") or sys.platform == "darwin": netcdf_base, netcdf_libs, netcdf_inc = get_netcdf_libs() - # print("netcdf include dir (line 188)", netcdf_inc) netcdf_lib_files = [] elif sys.platform in ("win32"): # but for now, still using our shipped libs. -# elif sys.platform in ("darwin", "win32"): third_party_dir = os.path.join('..', 'third_party_lib') if 'CONDA_PREFIX' in os.environ: netcdf_libs, netcdf_inc, netcdf_dll = get_conda_includes_win() @@ -218,7 +214,6 @@ def get_conda_includes_win(): netcdf_lib_files = [os.path.join(netcdf_libs, libfile.format(l)) for l in netcdf_names] else: - # the netCDF environment netcdf_base = os.path.join(third_party_dir, 'netcdf-4.3', sys.platform, architecture) @@ -341,7 +336,7 @@ def get_conda_includes_win(): ] -cpp_code_dir = os.path.join('..', 'lib_gnome') +cpp_code_dir = os.path.join('.', 'lib_gnome') cpp_files = [os.path.join(cpp_code_dir, f) for f in cpp_files] @@ -352,9 +347,10 @@ def get_conda_includes_win(): # Build the extension objects # suppressing certain warnings -compile_args = ["-Wno-unused-function", # unused function - cython creates a lot - "-stdlib=libc++", # to use the "correct" C++ libs - ] +compile_args = [ + "-Wno-unused-function", # unused function - cython creates a lot + "-stdlib=libc++", # to use the "correct" C++ libs +] extensions = [] @@ -380,11 +376,10 @@ def get_conda_includes_win(): # conda netcdf... if sys.platform == "darwin": - # on the mac, the libgnome code is linked into the cy_basic_types + # On the mac, the libgnome code is linked into the cy_basic_types # extension -- which makes it available to all the other extensions #. as long as it's imported first. # This is being done in the gnome/cy_gnome/__init__.py - basic_types_ext = Extension('gnome.cy_gnome.cy_basic_types', ['gnome/cy_gnome/cy_basic_types.pyx'] + cpp_files, language='c++', @@ -397,7 +392,6 @@ def get_conda_includes_win(): extensions.append(basic_types_ext) static_lib_files = [] - elif sys.platform == "win32": # On windows, the # build our compile arguments @@ -428,7 +422,7 @@ def get_conda_includes_win(): # we will reference this library when building all other extensions # fixme: Where is this getting built?? - # That filename needs to be dynamically determined with code somehat like: + # That filename needs to be dynamically determined with code somewhat like: # if sys.version_info.major > 2: # suffix = importlib.machinery.EXTENSION_SUFFIXES[0] # libname = 'gnome' + suffix @@ -470,7 +464,6 @@ def get_conda_includes_win(): language='c++', define_macros=macros, libraries=['netcdf'], - # include_dirs=[cpp_code_dir], include_dirs=include_dirs, )]) @@ -531,7 +524,6 @@ def get_conda_includes_win(): for mod_name in extension_names: cy_file = os.path.join("gnome/cy_gnome", mod_name + ".pyx") print("setting up cython extensions") - # print("include_dirs:", include_dirs) extensions.append(Extension('gnome.cy_gnome.' + mod_name, [cy_file], language="c++", @@ -574,17 +566,6 @@ def get_conda_includes_win(): language="c", )) -def get_version(): - """ - return the version number from the __init__ - """ - for line in open("gnome/__init__.py"): - if line.startswith("__version__"): - version = line.strip().split('=')[1].strip().strip("'").strip('"') - return version - raise ValueError("can't find version string in __init__") - - for e in extensions: e.cython_directives = {'language_level': "3"} # all are Python-3 @@ -603,25 +584,6 @@ def get_version(): requires=[], # want other packages here? cmdclass={'build_ext': build_ext, 'cleanall': cleanall}, - - # scripts, - - # metadata for upload to PyPI - author="Gnome team at NOAA/ORR/ERD", - author_email="orr.gnome@noaa.gov", - description=("GNOME (General NOAA Operational Modeling Environment) " - "is the modeling tool the Office of Response and " - "Restoration's (OR&R) Emergency Response Division uses to " - "predict the possible route, or trajectory, a pollutant " - "might follow in or on a body of water, such as in an " - "oil spill. " - "It can also be used as a customizable general particle " - "tracking code. " - "Branch: {} " - "LastUpdate: {}" - .format(branch_name, last_update)), - license="Public Domain", - keywords="oilspill modeling particle_tracking", url="https://github.com/NOAA-ORR-ERD/PyGnome" ) diff --git a/py_gnome/tests/unit_tests/test_model.py b/py_gnome/tests/unit_tests/test_model.py index 1196dcd33..da9d296b9 100644 --- a/py_gnome/tests/unit_tests/test_model.py +++ b/py_gnome/tests/unit_tests/test_model.py @@ -60,6 +60,7 @@ def test_exceptions(): wind = constant_wind(10, 270, units='knots') water = Water() model.weatherers += Evaporation(water, wind) + model.spills += Spill(Release(release_time=model.start_time), substance=test_oil) with raises(GnomeRuntimeError): model.check_inputs() diff --git a/py_gnome/tests/unit_tests/test_movers/test_cats_mover.py b/py_gnome/tests/unit_tests/test_movers/test_cats_mover.py index 29e63abfe..a602a3ce2 100644 --- a/py_gnome/tests/unit_tests/test_movers/test_cats_mover.py +++ b/py_gnome/tests/unit_tests/test_movers/test_cats_mover.py @@ -94,6 +94,19 @@ def run_loop(): return delta +def run_no_tide_loop(): + """ + test one time step with no tide attached to the mover + will check move is different from mover with tide. + also checks the motion is same for all LEs + """ + + pSpill = sample_sc_release(num_le, start_pos, rel_time) + cats = CatsMover(curr_file) + delta = _certain_loop(pSpill, cats) + + return delta + def test_uncertain_loop(): """ test one time step with uncertainty on the spill @@ -133,10 +146,23 @@ def test_certain_uncertain(): assert np.all(delta[:, :2] != u_delta[:, :2]) assert np.all(delta[:, 2] == u_delta[:, 2]) +def test_tide_notide(): + """ + make sure cats with tide and cats with no tide results in different deltas + """ + + tide_delta = run_loop() + no_tide_delta = run_no_tide_loop() + print() + print(tide_delta) + print(no_tide_delta) + assert np.all(tide_delta[:, :2] != no_tide_delta[:, :2]) + assert np.all(tide_delta[:, 2] == no_tide_delta[:, 2]) + c_cats = CatsMover(curr_file) -0 + def test_default_props(): """ test default properties diff --git a/py_gnome/tests/unit_tests/test_outputters/test_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_outputter.py index 05f4722c2..94dc88027 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_outputter.py @@ -155,8 +155,7 @@ def test_output_timestep(model, model_ts, output_ts): def test_bad_output_timestep(): """ - output timestep can never be zero - output timestep should match sign of model_time_step + output timestep can never be less than zero """ # on init: with pytest.raises(ValueError): @@ -164,23 +163,23 @@ def test_bad_output_timestep(): # changing it later out = Outputter(output_timestep=gs.hours(3)) - # allowing negative timestep for backwards run - #with pytest.raises(ValueError): - #out.output_timestep = gs.hours(-1) - assert out.output_timestep == gs.hours(3) - - out.output_timestep = gs.hours(-1) - with pytest.warns(RuntimeWarning, match="is less than zero") as warning: - out.prepare_for_model_run(model_start_time=datetime.now(),model_time_step=900) - - assert out.output_timestep == gs.hours(-1) + with pytest.raises(ValueError): + out.output_timestep = gs.hours(-1) - out.output_timestep = gs.hours(1) - with pytest.warns(RuntimeWarning, match="is greater than zero") as warning: - out.prepare_for_model_run(model_start_time=datetime.now(),model_time_step=-900) + assert out.output_timestep == gs.hours(3) - assert out.output_timestep == gs.hours(1) +# out.output_timestep = gs.hours(-1) +# with pytest.warns(RuntimeWarning, match="is less than zero") as warning: +# out.prepare_for_model_run(model_start_time=datetime.now(),model_time_step=900) +# +# assert out.output_timestep == gs.hours(-1) +# +# out.output_timestep = gs.hours(1) +# with pytest.warns(RuntimeWarning, match="is greater than zero") as warning: +# out.prepare_for_model_run(model_start_time=datetime.now(),model_time_step=-900) +# +# assert out.output_timestep == gs.hours(1) def test_too_small_output_timestep(model): diff --git a/py_gnome/tests/unit_tests/test_spills/test_release.py b/py_gnome/tests/unit_tests/test_spills/test_release.py index 13b124bfc..7914ac7ce 100644 --- a/py_gnome/tests/unit_tests/test_spills/test_release.py +++ b/py_gnome/tests/unit_tests/test_spills/test_release.py @@ -198,6 +198,46 @@ def test_serialization(self, r1): deser = PointLineRelease.deserialize(ser) assert deser == r1 + def test_LE_initialization_instantaneous(self, r2): + #initialize_LEs(self, to_rel, data, current_time, time_step) + data = LEData() + ts = 900 + r = r2 + num_le = 0 + r.prepare_for_model_run(ts) + + data.prepare_for_model_run(r.array_types, None) + data.extend_data_arrays(10) + #initialize over the time interval 0-10% + r.initialize_LEs(10, data, r.release_time, r.release_time+timedelta(seconds=ts)) + + #particles should have positions spread over the + #line from start_position to end_position + assert len(data['positions']) == 10 + for pos in data['positions']: + for d in [0,1,2]: + #instantaneous so particles should be spread across whole line + assert pos[d] >= r.start_position[d] + num_le + num_le += 1 + + assert np.all(data['mass'] == r._mass_per_le) + + #reset and try overlap beginning + data.rewind() + num_le = 0 + data.prepare_for_model_run(r.array_types, None) + data.extend_data_arrays(100) + #initialize 100 LEs overlapping the start of the release + r.initialize_LEs(100, data, r.release_time - timedelta(seconds=ts/2), r.release_time) + for pos in data['positions']: + for d in [0,1,2]: + #instantaneous so particles should be spread across whole line + assert pos[d] >= r.start_position[d] + num_le*.1 + num_le += 1 + + assert np.all(data['mass'] == r._mass_per_le) + + #This isn't supported yet in pytest???? #@pytest.mark.parametrize('r', [r1, r3]) def test_LE_initialization(self, r1, r3): diff --git a/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py b/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py index 7bf89be89..66712460e 100644 --- a/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py +++ b/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py @@ -16,13 +16,60 @@ from gnome.environment import constant_wind, Water, Wind from gnome.weatherers import Evaporation from gnome.outputters import WeatheringOutput -from gnome.basic_types import oil_status +from gnome.basic_types import oil_status, datetime_value_2d from .conftest import weathering_data_arrays, test_oil from ..conftest import (# sample_model, sample_model_weathering) +def test_evaporation_bad_wind(): + series = np.zeros((2, ), dtype=datetime_value_2d) + start_time = datetime(2000, 1, 1, 1) # same as conftest time + + series[0] = (start_time + timedelta(hours=24), (10, 45)) # starts after model start + series[1] = (start_time + timedelta(hours=48), (10, 90)) + wind = Wind(timeseries=series, units='m/s') + wind.extrapolate=True # add separate + + evap = Evaporation(Water(), wind=wind) + (sc, time_step) = weathering_data_arrays(evap.array_types, evap.water)[:2] + + model_time = (sc.spills[0].release_time + + timedelta(seconds=time_step)) + + evap.prepare_for_model_run(sc) + evap.prepare_for_model_step(sc, time_step, model_time) + with pytest.raises(ValueError)as excinfo: + evap.weather_elements(sc, time_step, model_time) + assert "not within the bounds" in str(excinfo.value) + +@pytest.mark.xfail +def test_evaporation_extrapolate_bad_wind(): + """ + NOTE: This test should work once we allow extrapolation for weatherers + """ + series = np.zeros((2, ), dtype=datetime_value_2d) + start_time = datetime(2000, 1, 1, 1) # same as conftest time + + series[0] = (start_time + timedelta(hours=24), (10, 45)) # starts after model start + series[1] = (start_time + timedelta(hours=48), (10, 90)) + wind = Wind(timeseries=series, units='m/s', extrapolation_is_allowed=True) + + evap = Evaporation(Water(), wind=wind) + (sc, time_step) = weathering_data_arrays(evap.array_types, evap.water)[:2] + + model_time = (sc.spills[0].release_time + + timedelta(seconds=time_step)) + + evap.prepare_for_model_run(sc) + evap.prepare_for_model_step(sc, time_step, model_time) + evap.weather_elements(sc, time_step, model_time) + for spill in sc.spills: + mask = sc.get_spill_mask(spill) + assert np.all(sc['evap_decay_constant'][mask, :] < 0.0) + + def test_evaporation_no_wind(): evap = Evaporation(Water(), wind=constant_wind(0., 0)) (sc, time_step) = weathering_data_arrays(evap.array_types, evap.water)[:2]