From 4f8fba4b69a4836da8e6f300a3dfebc2b8a63df2 Mon Sep 17 00:00:00 2001 From: Micah Sandusky <32111103+micah-prime@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:57:02 -0600 Subject: [PATCH] API for Point and Layer data (#81) * Start creating classes to act as stable api * get the shapefile filtering working for points. Start on raster code * Add some tests and comments for the api * Test the query methods * Add tests for from area both for polygon and from point and buffer * start layer tests, not passing yet * Finish Layer data tests. Add more doc strings. Modify README for testing * site names attribute * add two notebooks of examples * rename notebook * use plotly for interactive plots * Working on documentation for the API * Further documentation of the api methodology * typo * fix the box plot in the api example * errors in documentation * add date filtering * Add line about date filtering * Update readthedocs configuration for updated template schema. Add plotly to docs reqs. Add template for plotly js cdn * Make sure raster queries are limited * better list compare * try using python3 pip * is setuptools the issue? * we don't need to install docs reqs for github builds * more req work * update dev reqs * Fix database to use local for testing. I think we have a reqs issue. Now conversions to geopandas are failing locally when the API tests passed in general before * briefly use the DB for testing * use sqlalchemy >= 2 * working on test fix * Install doc reqs * Can't install sphinx 7.3 on python 3.8 * sphinx in req_dev is getting us * try getting rid of matplotlib req * try adding a make clean * what if we just let pip do the work? * setup_requires is deprecated. Could move to pyproj.toml, but do we need it? --- .github/workflows/build.yml | 8 +- .github/workflows/main.yml | 8 +- .readthedocs.yaml | 26 +- README.rst | 35 +- docs/_templates/layout.html | 6 + docs/api.rst | 109 +- docs/gallery/api_intro_example.ipynb | 1474 ++++++++ .../api_plot_pit_density_example.ipynb | 3023 +++++++++++++++++ docs/requirements.txt | 6 +- requirements.txt | 2 +- requirements_dev.txt | 12 +- setup.py | 10 +- snowexsql/api.py | 328 ++ snowexsql/conversions.py | 4 +- snowexsql/db.py | 2 +- tests/map.html | 147 + tests/scratch.py | 35 + tests/test_api.py | 250 ++ tests/test_db.py | 2 + 19 files changed, 5442 insertions(+), 45 deletions(-) create mode 100644 docs/_templates/layout.html create mode 100644 docs/gallery/api_intro_example.ipynb create mode 100644 docs/gallery/api_plot_pit_density_example.ipynb create mode 100644 snowexsql/api.py create mode 100644 tests/map.html create mode 100644 tests/scratch.py create mode 100644 tests/test_api.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65a63aa..5f6326c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macOS-latest] - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -27,9 +27,9 @@ jobs: - name: Install Macos/Linux dependencies run: | - pip install --upgrade pip setuptools wheel - python -m pip install -r requirements.txt - python setup.py install + python3 -m pip install --upgrade pip setuptools wheel + make clean + python3 -m pip install . - name: Install Validation run: | python -c "import snowexsql" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca80746..16de730 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, '3.10'] services: @@ -44,9 +44,9 @@ jobs: run: | sudo apt-get update sudo apt-get install -y postgis gdal-bin - python -m pip install --upgrade pip - pip install pytest coverage - if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + python3 -m pip install --upgrade pip + python3 -m pip install pytest coverage + if [ -f requirements_dev.txt ]; then python3 -m pip install -r requirements_dev.txt; fi - name: Test with pytest run: | pytest -s diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6d043c7..69763a0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,13 +1,17 @@ - version: 2 +version: 2 - sphinx: - configuration: docs/conf.py - fail_on_warning: false +sphinx: + configuration: docs/conf.py + fail_on_warning: false - python: - version: 3.8 - install: - - requirements: docs/requirements.txt - - requirements: requirements.txt - - method: setuptools - path: . +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +python: + install: + - requirements: docs/requirements.txt + - requirements: requirements.txt + - method: setuptools + path: . diff --git a/README.rst b/README.rst index b31d0b8..acc80a1 100644 --- a/README.rst +++ b/README.rst @@ -74,14 +74,47 @@ If you are using `conda` you may need to reinstall the following using conda: * Jupyter notebook * nbconvert + +I want data fast! +----------------- +A programmatic API has been created for fast and standard +access to Point and Layer data. There are two examples_ covering the +features and usage of the api. See the specific api_ documentation for +detailed description. + +.. _api: https://snowexsql.readthedocs.io/en/latest/api.html + +.. code-block:: python + + from snowexsql.api import PointMeasurements, LayerMeasurements + # The main functions we will use are `from_area` and `from_filter` like this + df = PointMeasurements.from_filter( + date=date(2020, 5, 28), instrument='camera' + ) + print(df.head()) + Tests ----- +Before testing, in a seperate terminal, we need to run a local instance +of the database. This can be done with + +.. code-block:: bash + + docker-compose up -d + +When you are finished testing, make sure to turn the docker off + +.. code-block:: bash + + docker-compose down + + Quickly test your installation by running: .. code-block:: bash - pytest + python3 -m pytest tests/ The goal of this project is to have high fidelity in data interpretation/submission to the database. To see the current diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000..878c145 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,6 @@ +{% extends "!layout.html" %} + +{% block footer %} + +{{ super() }} +{% endblock %} diff --git a/docs/api.rst b/docs/api.rst index 0f95856..59c1fc7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,10 +1,111 @@ API Documentation ================= +.. role:: python(code) + :language: python -Information on snowexsql functions, classes, and modules. +Background +---------- +The API (not a rest API, more of an SDK) is a set of python classes +designed for easy and standardized access to the database data. -.. toctree:: - :maxdepth: 4 +The classes can both describe what data is available, and return +data in a GeoPandas dataframe. - snowexsql +Components +---------- +There are two main API classes for data access. +.. code-block:: python + + from snowexsql.api import PointMeasurements, LayerMeasurements + +:code:`PointMeasurements` gives access to the PointData (depths, GPR, etc), and +:code:`LayerMeasurements` gives access to the LayerData (pits, etc). + +Both of the classes have the same methods, although they access different +tables in the database. + +The primary methods for accessing data are :code:`.from_area` and +:code:`.from_filter`. Both of these methods return a GeoPandas dataframe. + +.from_filter +------------ + +The :code:`.from_filter` is the simpler of the two search methods. It takes in +a variety of key word args (kwargs) and returns a dataset that meets +all of the criteria. + +.. code-block:: python + + df = LayerMeasurements.from_filter( + type="density", + site_name="Boise River Basin", + limit=1000 + ) + +In this example, we filter to all the layer measurements of `density` +that were taken in the `Boise River Basin`, and we `limit` to the top +1000 measurements. + +Each kwarg (except date) **can take in a list or a single value** so you could change +this to :code:`site_name=["Boise River Basin", "Grand Mesa"]` + +To find what `kwargs` are allowed, we can check the class + +.. code-block:: python + + LayerMeasurements.ALLOWED_QRY_KWARGS + +For :code:`LayerMeasurements` this will return +:code:`["site_name", "site_id", "date", "instrument", "observers", "type", "utm_zone", "pit_id", "date_greater_equal", "date_less_equal"]` + +so we can filter by any of these as inputs to the function. + +**Notice `limit` is not specified here**. Limit is in the :code:`SPECIAL_KWARGS` +and gets handled at the end of the query. + +**Notice `date_greater_equal` and `date_less_equal`** for filtering the `date` +parameter using `>=` and `<=` logic. + +To find what values are allowed for each, we can check the propeties of the +class. Both :code:`LayerMeasurements` and :code:`PointMeasurements` have +the following properties. + + * all_site_names + * all_types + * all_dates + * all_observers + * all_instruments + +So you can find all the instruments for filtering like :code:`LayerMeasurements().all_instruments`. +**Note** - these must be called from an instantiated class like shown earlier +in this line. + +.from_area +---------- + +The signature for :code:`.from_area` looks like this + +.. code-block:: python + + def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs): + +It is a class method, so it *does not need an instantiated class*. +The :code:`**kwargs` argument takes the same inputs as the :code:`from_filter` +function. + +The big difference is that from area will filter to results either within +:code:`shp` (a `shapely` polygon) **or** within :code:`buffer` radius +around :code:`pt` (a `shapely` point). + + +Large Query Exception and Limit +------------------------------- + +By default, if more than 1000 records will be returned, and **no limit** +is provided. The query will fail. This is intentional so that we are aware +of large queries. If you understand your query will be large and need +more than 1000 records returned, add a :code:`limit` kwarg to your query +with a value greater than the number you need returned. +**This will override the default behavior** and return as many records as +you requested. diff --git a/docs/gallery/api_intro_example.ipynb b/docs/gallery/api_intro_example.ipynb new file mode 100644 index 0000000..79cf441 --- /dev/null +++ b/docs/gallery/api_intro_example.ipynb @@ -0,0 +1,1474 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Welcome to the API!\n", + "\n", + "**Goal**: Easy programmatic access to the database with **no user SQL**\n", + "\n", + "\n", + "## Notes\n", + "\n", + " * This is not a REST API, more of an SDK\n", + " * Current access is for *point* and *layer* data\n", + " * Funtions return **lists** or **Geopandas Dataframes**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 1. Import the classes, explore them" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# imports\n", + "from datetime import date\n", + "import geopandas as gpd\n", + "from snowexsql.api import PointMeasurements, LayerMeasurements" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
site_namedatetime_createdtime_updatediddoidate_accessedinstrumenttypeunits...northingeastingelevationutm_zonegeomtimesite_idversion_numberequipmentvalue
0Grand Mesa2020-05-282022-06-30 22:58:59.800562+00:00None42443None2022-06-30cameradepthcm...4.321444e+06743766.479497None12POINT (743766.479 4321444.155)18:00:00+00:00NoneNonecamera id = W1B-2.99924
1Grand Mesa2020-05-282022-06-30 22:58:59.800562+00:00None42444None2022-06-30cameradepthcm...4.321444e+06743766.479497None12POINT (743766.479 4321444.155)19:00:00+00:00NoneNonecamera id = W1B1.50148
2Grand Mesa2020-05-282022-06-30 22:58:59.800562+00:00None43187None2022-06-30cameradepthcm...4.331951e+06249164.808618None13POINT (249164.809 4331951.003)18:00:00+00:00NoneNonecamera id = E9B-1.15255
3Grand Mesa2020-05-282022-06-30 22:58:59.800562+00:00None43188None2022-06-30cameradepthcm...4.331951e+06249164.808618None13POINT (249164.809 4331951.003)19:00:00+00:00NoneNonecamera id = E9B1.16381
4Grand Mesa2020-05-282022-06-30 22:58:59.800562+00:00None43189None2022-06-30cameradepthcm...4.331951e+06249164.808618None13POINT (249164.809 4331951.003)20:00:00+00:00NoneNonecamera id = E9B-2.31073
\n", + "

5 rows × 23 columns

\n", + "
" + ], + "text/plain": [ + " site_name date time_created time_updated \\\n", + "0 Grand Mesa 2020-05-28 2022-06-30 22:58:59.800562+00:00 None \n", + "1 Grand Mesa 2020-05-28 2022-06-30 22:58:59.800562+00:00 None \n", + "2 Grand Mesa 2020-05-28 2022-06-30 22:58:59.800562+00:00 None \n", + "3 Grand Mesa 2020-05-28 2022-06-30 22:58:59.800562+00:00 None \n", + "4 Grand Mesa 2020-05-28 2022-06-30 22:58:59.800562+00:00 None \n", + "\n", + " id doi date_accessed instrument type units ... northing \\\n", + "0 42443 None 2022-06-30 camera depth cm ... 4.321444e+06 \n", + "1 42444 None 2022-06-30 camera depth cm ... 4.321444e+06 \n", + "2 43187 None 2022-06-30 camera depth cm ... 4.331951e+06 \n", + "3 43188 None 2022-06-30 camera depth cm ... 4.331951e+06 \n", + "4 43189 None 2022-06-30 camera depth cm ... 4.331951e+06 \n", + "\n", + " easting elevation utm_zone geom \\\n", + "0 743766.479497 None 12 POINT (743766.479 4321444.155) \n", + "1 743766.479497 None 12 POINT (743766.479 4321444.155) \n", + "2 249164.808618 None 13 POINT (249164.809 4331951.003) \n", + "3 249164.808618 None 13 POINT (249164.809 4331951.003) \n", + "4 249164.808618 None 13 POINT (249164.809 4331951.003) \n", + "\n", + " time site_id version_number equipment value \n", + "0 18:00:00+00:00 None None camera id = W1B -2.99924 \n", + "1 19:00:00+00:00 None None camera id = W1B 1.50148 \n", + "2 18:00:00+00:00 None None camera id = E9B -1.15255 \n", + "3 19:00:00+00:00 None None camera id = E9B 1.16381 \n", + "4 20:00:00+00:00 None None camera id = E9B -2.31073 \n", + "\n", + "[5 rows x 23 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# The main functions we will use are `from_area` and `from_filter` like this\n", + "df = PointMeasurements.from_filter(\n", + " date=date(2020, 5, 28), instrument='camera'\n", + ")\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Notice:\n", + " * We did not need to manage SQL\n", + " * We got a geopandas array\n", + " * We filtered on specific attributes known to be in the database" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### How do I know what to filter by?" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['site_name', 'site_id', 'date', 'instrument', 'observers', 'type', 'utm_zone']\n", + "['site_name', 'site_id', 'date', 'instrument', 'observers', 'type', 'utm_zone', 'pit_id']\n" + ] + } + ], + "source": [ + "# Find what you can filter by\n", + "print(PointMeasurements.ALLOWED_QRY_KWARGS)\n", + "print(LayerMeasurements.ALLOWED_QRY_KWARGS)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### How do I know what values work for filtering?" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('Catherine Breen, Cassie Lumbrazo',), (None,), ('Ryan Webb',), ('Randall Bonnell',), ('Tate Meehan',)]\n" + ] + } + ], + "source": [ + "print(PointMeasurements().all_observers)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Try it out\n", + "\n", + "* What instrument could you filter by for PointData?\n", + "* What site names could you filter by for LayerData?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice we instantiate the class \n", + "`PointMeasurements()`\n", + "Before calling the property `.all_observers`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# " + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Explore the points\n", + "df.crs\n", + "df.to_crs(\"EPSG:4326\").loc[:,[\"id\", \"value\", \"type\", \"geom\", \"instrument\"]].explore()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### What if I have a point or a shapefile\n", + "\n", + "Both the PointMeasurement and LayerMeasurement class have a function called `from_area`\n", + "that takes either a `shapely` polygon or a `shapely` point and a radius as well as the same\n", + "filter kwargs available in `.from_filter`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Set up a fake shapefile\n", + "gdf = gpd.GeoDataFrame(\n", + " geometry=gpd.points_from_xy(\n", + " [743766.4794971556], [4321444.154620216], crs=\"epsg:26912\"\n", + " ).buffer(2000.0)\n", + ").set_crs(\"epsg:26912\")\n", + "\n", + "# This is the area we will filter to\n", + "gdf.explore()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get density near the point\n", + "df = LayerMeasurements.from_area(\n", + " type=\"density\",\n", + " shp=gdf.iloc[0].geometry,\n", + ")\n", + "\n", + "df.to_crs(\"EPSG:4326\").loc[:,[\"id\", \"depth\", \"value\", \"type\", \"geom\"]].explore()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How much filtering is enough? " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "I got a `LargeQueryCheckException`\n", + "\n", + "GIVE ME THE DATA PLEASE" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "tags": [ + "nbsphinx-gallery", + "nbsphinx-thumbnail" + ] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed query for PointData\n" + ] + }, + { + "ename": "LargeQueryCheckException", + "evalue": "Query will return 33364 number of records, but we have a default max of 1000. If you want to proceed, set the 'limit' filter to the desired number of records.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mLargeQueryCheckException\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/jh/tvv3prb117d22jyn0vmbjn880000gn/T/ipykernel_51325/2889856166.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# This query will fail\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m df = PointMeasurements.from_filter(\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0minstrument\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"magnaprobe\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m )\n", + "\u001b[0;32m~/projects/m3works/snowexsql/snowexsql/api.py\u001b[0m in \u001b[0;36mfrom_filter\u001b[0;34m(cls, **kwargs)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0mLOG\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Failed query for PointData\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/m3works/snowexsql/snowexsql/api.py\u001b[0m in \u001b[0;36mfrom_filter\u001b[0;34m(cls, **kwargs)\u001b[0m\n\u001b[1;32m 181\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 182\u001b[0m \u001b[0mqry\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMODEL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 183\u001b[0;31m \u001b[0mqry\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend_qry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqry\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 184\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquery_to_geopandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqry\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/m3works/snowexsql/snowexsql/api.py\u001b[0m in \u001b[0;36mextend_qry\u001b[0;34m(cls, qry, check_size, **kwargs)\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcheck_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqry\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mqry\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/m3works/snowexsql/snowexsql/api.py\u001b[0m in \u001b[0;36m_check_size\u001b[0;34m(cls, qry, kwargs)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mqry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMAX_RECORD_COUNT\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"limit\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m raise LargeQueryCheckException(\n\u001b[0m\u001b[1;32m 72\u001b[0m \u001b[0;34mf\"Query will return {count} number of records,\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 73\u001b[0m \u001b[0;34mf\" but we have a default max of {cls.MAX_RECORD_COUNT}.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mLargeQueryCheckException\u001b[0m: Query will return 33364 number of records, but we have a default max of 1000. If you want to proceed, set the 'limit' filter to the desired number of records." + ] + } + ], + "source": [ + "# This query will fail\n", + "df = PointMeasurements.from_filter(\n", + " instrument=\"magnaprobe\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
site_namedatetime_createdtime_updatediddoidate_accessedinstrumenttypeunits...northingeastingelevationutm_zonegeomtimesite_idversion_numberequipmentvalue
0Grand Mesa2020-01-292022-06-30 22:56:52.635035+00:00None8713https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.322865e+06741881.1024663037.812POINT (741881.102 4322865.037)14:56:00+00:00None1CRREL_C85.0
1Grand Mesa2020-01-292022-06-30 22:56:52.635035+00:00None8714https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.322859e+06741878.6753803038.012POINT (741878.675 4322859.408)14:57:00+00:00None1CRREL_C72.0
2Grand Mesa2020-01-292022-06-30 22:56:52.635035+00:00None8715https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.322855e+06741877.0800583037.112POINT (741877.080 4322854.914)14:57:00+00:00None1CRREL_C84.0
3Grand Mesa2020-01-292022-06-30 22:56:52.635035+00:00None8716https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.322850e+06741875.4847333035.512POINT (741875.485 4322850.421)14:57:00+00:00None1CRREL_C84.0
4Grand Mesa2020-01-292022-06-30 22:56:52.635035+00:00None8717https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.322845e+06741873.9235123034.612POINT (741873.924 4322844.818)14:57:00+00:00None1CRREL_C78.0
\n", + "

5 rows × 23 columns

\n", + "
" + ], + "text/plain": [ + " site_name date time_created time_updated id \\\n", + "0 Grand Mesa 2020-01-29 2022-06-30 22:56:52.635035+00:00 None 8713 \n", + "1 Grand Mesa 2020-01-29 2022-06-30 22:56:52.635035+00:00 None 8714 \n", + "2 Grand Mesa 2020-01-29 2022-06-30 22:56:52.635035+00:00 None 8715 \n", + "3 Grand Mesa 2020-01-29 2022-06-30 22:56:52.635035+00:00 None 8716 \n", + "4 Grand Mesa 2020-01-29 2022-06-30 22:56:52.635035+00:00 None 8717 \n", + "\n", + " doi date_accessed instrument type \\\n", + "0 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "1 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "2 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "3 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "4 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "\n", + " units ... northing easting elevation utm_zone \\\n", + "0 cm ... 4.322865e+06 741881.102466 3037.8 12 \n", + "1 cm ... 4.322859e+06 741878.675380 3038.0 12 \n", + "2 cm ... 4.322855e+06 741877.080058 3037.1 12 \n", + "3 cm ... 4.322850e+06 741875.484733 3035.5 12 \n", + "4 cm ... 4.322845e+06 741873.923512 3034.6 12 \n", + "\n", + " geom time site_id version_number \\\n", + "0 POINT (741881.102 4322865.037) 14:56:00+00:00 None 1 \n", + "1 POINT (741878.675 4322859.408) 14:57:00+00:00 None 1 \n", + "2 POINT (741877.080 4322854.914) 14:57:00+00:00 None 1 \n", + "3 POINT (741875.485 4322850.421) 14:57:00+00:00 None 1 \n", + "4 POINT (741873.924 4322844.818) 14:57:00+00:00 None 1 \n", + "\n", + " equipment value \n", + "0 CRREL_C 85.0 \n", + "1 CRREL_C 72.0 \n", + "2 CRREL_C 84.0 \n", + "3 CRREL_C 84.0 \n", + "4 CRREL_C 78.0 \n", + "\n", + "[5 rows x 23 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Th queries will pass\n", + "df = PointMeasurements.from_filter(\n", + " instrument=\"magnaprobe\",\n", + " limit=100\n", + ")\n", + "\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### DANGER ZONE\n", + "If you need more than 1000 points returned, you can specify so with the `limit`\n", + "\n", + "The intention is to be aware of how much data will be returned" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
site_namedatetime_createdtime_updatediddoidate_accessedinstrumenttypeunits...northingeastingelevationutm_zonegeomtimesite_idversion_numberequipmentvalue
0Grand Mesa2020-01-282022-06-30 22:56:52.635035+00:00None4663https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.323938e+06747760.1274173143.912POINT (747760.127 4323937.874)20:22:00+00:00None1CRREL_B64.0
1Grand Mesa2020-01-282022-06-30 22:56:52.635035+00:00None4102https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.324060e+06747975.5332293151.812POINT (747975.533 4324060.214)18:48:00+00:00None1CRREL_B106.0
2Grand Mesa2020-01-282022-06-30 22:56:52.635035+00:00None4103https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.324058e+06747973.0058693153.812POINT (747973.006 4324057.912)18:48:00+00:00None1CRREL_B110.0
3Grand Mesa2020-01-282022-06-30 22:56:52.635035+00:00None4104https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.324057e+06747973.0408483153.512POINT (747973.041 4324056.802)18:48:00+00:00None1CRREL_B106.0
4Grand Mesa2020-01-282022-06-30 22:56:52.635035+00:00None4105https://doi.org/10.5067/9IA978JIACAR2022-06-30magnaprobedepthcm...4.324055e+06747972.2450323154.012POINT (747972.245 4324054.555)18:48:00+00:00None1CRREL_B107.0
\n", + "

5 rows × 23 columns

\n", + "
" + ], + "text/plain": [ + " site_name date time_created time_updated id \\\n", + "0 Grand Mesa 2020-01-28 2022-06-30 22:56:52.635035+00:00 None 4663 \n", + "1 Grand Mesa 2020-01-28 2022-06-30 22:56:52.635035+00:00 None 4102 \n", + "2 Grand Mesa 2020-01-28 2022-06-30 22:56:52.635035+00:00 None 4103 \n", + "3 Grand Mesa 2020-01-28 2022-06-30 22:56:52.635035+00:00 None 4104 \n", + "4 Grand Mesa 2020-01-28 2022-06-30 22:56:52.635035+00:00 None 4105 \n", + "\n", + " doi date_accessed instrument type \\\n", + "0 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "1 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "2 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "3 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "4 https://doi.org/10.5067/9IA978JIACAR 2022-06-30 magnaprobe depth \n", + "\n", + " units ... northing easting elevation utm_zone \\\n", + "0 cm ... 4.323938e+06 747760.127417 3143.9 12 \n", + "1 cm ... 4.324060e+06 747975.533229 3151.8 12 \n", + "2 cm ... 4.324058e+06 747973.005869 3153.8 12 \n", + "3 cm ... 4.324057e+06 747973.040848 3153.5 12 \n", + "4 cm ... 4.324055e+06 747972.245032 3154.0 12 \n", + "\n", + " geom time site_id version_number \\\n", + "0 POINT (747760.127 4323937.874) 20:22:00+00:00 None 1 \n", + "1 POINT (747975.533 4324060.214) 18:48:00+00:00 None 1 \n", + "2 POINT (747973.006 4324057.912) 18:48:00+00:00 None 1 \n", + "3 POINT (747973.041 4324056.802) 18:48:00+00:00 None 1 \n", + "4 POINT (747972.245 4324054.555) 18:48:00+00:00 None 1 \n", + "\n", + " equipment value \n", + "0 CRREL_B 64.0 \n", + "1 CRREL_B 106.0 \n", + "2 CRREL_B 110.0 \n", + "3 CRREL_B 106.0 \n", + "4 CRREL_B 107.0 \n", + "\n", + "[5 rows x 23 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# DANGER ZONE\n", + "# If you need more than 1000 points returned, you can specify so with the limit\n", + "df = PointMeasurements.from_filter(\n", + " date=date(2020, 1, 28),\n", + " instrument=\"magnaprobe\",\n", + " limit=3000\n", + ")\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# THE END\n", + "\n", + "### Go forth and explore" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/gallery/api_plot_pit_density_example.ipynb b/docs/gallery/api_plot_pit_density_example.ipynb new file mode 100644 index 0000000..86647af --- /dev/null +++ b/docs/gallery/api_plot_pit_density_example.ipynb @@ -0,0 +1,3023 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Welcome to the API PT2!\n", + "\n", + "## Data Edition\n", + "\n", + "#### Goal - Filter down to the pit density we want and plot it" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 1. Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# imports\n", + "from datetime import date\n", + "import geopandas as gpd\n", + "import matplotlib.pyplot as plt\n", + "from snowexsql.api import PointMeasurements, LayerMeasurements" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2. Find the pits in the Boise River Basin" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('Cameron Pass',), ('Sagehen Creek',), ('Fraser Experimental Forest',), ('Mammoth Lakes',), ('Niwot Ridge',), ('Boise River Basin',), ('Little Cottonwood Canyon',), ('East River',), ('American River Basin',), ('Senator Beck',), ('Jemez River',), ('Grand Mesa',)]\n" + ] + } + ], + "source": [ + "# Find site names we can use\n", + "print(LayerMeasurements().all_site_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get the first 1000 measurements from the Boise River Basin Site\n", + "df = LayerMeasurements.from_filter(\n", + " type=\"density\",\n", + " site_name=\"Boise River Basin\",\n", + " limit=1000\n", + ")\n", + "\n", + "# Explore the pits so we can find an interesting site\n", + "df.loc[:, [\"site_id\", \"geom\"]].drop_duplicates().explore()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3. Pick a point of interest" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [ + "nbsphinx-gallery", + "nbsphinx-thumbnail" + ] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# We noticed there are a lot of pits (timeseries pits) for Banner Open\n", + "# Filter down to ONE timeseries\n", + "site_id = \"Banner Open\"\n", + "df = LayerMeasurements.from_filter(\n", + " type=\"density\",\n", + " site_id=site_id\n", + ").set_crs(\"epsg:26911\")\n", + "\n", + "df.loc[:, [\"site_id\", \"geom\"]].drop_duplicates().explore()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idlatitudelongitudenorthingeastingutm_zonedepthbottom_depthvalue
date
2019-12-1822609.544.30464-115.236034.907222e+06640699.66612111.026.00000016.000000279.000000
2020-01-0922670.044.30463-115.236014.907221e+06640701.28528911.051.00000041.000000156.777778
2020-01-2322746.044.30461-115.235984.907219e+06640703.72598811.061.72727351.727273254.166667
2020-01-3022830.544.30461-115.235984.907219e+06640703.72598811.072.00000062.000000236.000000
2020-02-0622910.044.30458-115.235944.907216e+06640706.98822211.072.00000062.000000254.141026
\n", + "
" + ], + "text/plain": [ + " id latitude longitude northing easting \\\n", + "date \n", + "2019-12-18 22609.5 44.30464 -115.23603 4.907222e+06 640699.666121 \n", + "2020-01-09 22670.0 44.30463 -115.23601 4.907221e+06 640701.285289 \n", + "2020-01-23 22746.0 44.30461 -115.23598 4.907219e+06 640703.725988 \n", + "2020-01-30 22830.5 44.30461 -115.23598 4.907219e+06 640703.725988 \n", + "2020-02-06 22910.0 44.30458 -115.23594 4.907216e+06 640706.988222 \n", + "\n", + " utm_zone depth bottom_depth value \n", + "date \n", + "2019-12-18 11.0 26.000000 16.000000 279.000000 \n", + "2020-01-09 11.0 51.000000 41.000000 156.777778 \n", + "2020-01-23 11.0 61.727273 51.727273 254.166667 \n", + "2020-01-30 11.0 72.000000 62.000000 236.000000 \n", + "2020-02-06 11.0 72.000000 62.000000 254.141026 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get the mean of each date sampled\n", + "df[\"value\"] = df[\"value\"].astype(float)\n", + "df.set_index(\"date\", inplace=True)\n", + "mean_values = df.groupby(df.index).mean()\n", + "mean_values.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Notes on this mean\n", + "\n", + "Taking this `mean` as bulk density **could be flawed** if layers are overlapping or layers vary in thickness" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install plotly\n", + "import plotly.express as px\n", + "# For rendering in readthedocs\n", + "import plotly.offline as py\n", + "py.init_notebook_mode(connected=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hovertemplate": "Date=%{x}
Density=%{y}", + "legendgroup": "", + "line": { + "color": "#636efa", + "dash": "solid" + }, + "marker": { + "symbol": "circle" + }, + "mode": "lines", + "name": "", + "orientation": "v", + "showlegend": false, + "type": "scatter", + "x": [ + "2019-12-18", + "2020-01-09", + "2020-01-23", + "2020-01-30", + "2020-02-06", + "2020-02-13", + "2020-02-19", + "2020-02-27", + "2020-03-05", + "2020-03-12" + ], + "xaxis": "x", + "y": [ + 279, + 156.77777777777777, + 254.16666666666669, + 236, + 254.14102564102566, + 276.1666666666667, + 274.76388888888886, + 295.04545454545456, + 319.93939393939394, + 323.95454545454544 + ], + "yaxis": "y" + } + ], + "layout": { + "legend": { + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#f2f5fa" + }, + "error_y": { + "color": "#f2f5fa" + }, + "marker": { + "line": { + "color": "rgb(17,17,17)", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "rgb(17,17,17)", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#A2B1C6", + "gridcolor": "#506784", + "linecolor": "#506784", + "minorgridcolor": "#506784", + "startlinecolor": "#A2B1C6" + }, + "baxis": { + "endlinecolor": "#A2B1C6", + "gridcolor": "#506784", + "linecolor": "#506784", + "minorgridcolor": "#506784", + "startlinecolor": "#A2B1C6" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "marker": { + "line": { + "color": "#283442" + } + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "line": { + "color": "#283442" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#506784" + }, + "line": { + "color": "rgb(17,17,17)" + } + }, + "header": { + "fill": { + "color": "#2a3f5f" + }, + "line": { + "color": "rgb(17,17,17)" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#f2f5fa", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#f2f5fa" + }, + "geo": { + "bgcolor": "rgb(17,17,17)", + "lakecolor": "rgb(17,17,17)", + "landcolor": "rgb(17,17,17)", + "showlakes": true, + "showland": true, + "subunitcolor": "#506784" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "dark" + }, + "paper_bgcolor": "rgb(17,17,17)", + "plot_bgcolor": "rgb(17,17,17)", + "polar": { + "angularaxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + }, + "bgcolor": "rgb(17,17,17)", + "radialaxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "rgb(17,17,17)", + "gridcolor": "#506784", + "gridwidth": 2, + "linecolor": "#506784", + "showbackground": true, + "ticks": "", + "zerolinecolor": "#C8D4E3" + }, + "yaxis": { + "backgroundcolor": "rgb(17,17,17)", + "gridcolor": "#506784", + "gridwidth": 2, + "linecolor": "#506784", + "showbackground": true, + "ticks": "", + "zerolinecolor": "#C8D4E3" + }, + "zaxis": { + "backgroundcolor": "rgb(17,17,17)", + "gridcolor": "#506784", + "gridwidth": 2, + "linecolor": "#506784", + "showbackground": true, + "ticks": "", + "zerolinecolor": "#C8D4E3" + } + }, + "shapedefaults": { + "line": { + "color": "#f2f5fa" + } + }, + "sliderdefaults": { + "bgcolor": "#C8D4E3", + "bordercolor": "rgb(17,17,17)", + "borderwidth": 1, + "tickwidth": 0 + }, + "ternary": { + "aaxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + }, + "baxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + }, + "bgcolor": "rgb(17,17,17)", + "caxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "updatemenudefaults": { + "bgcolor": "#506784", + "borderwidth": 0 + }, + "xaxis": { + "automargin": true, + "gridcolor": "#283442", + "linecolor": "#506784", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "#283442", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "#283442", + "linecolor": "#506784", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "#283442", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Mean Density Banner Open" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Date" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Density" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Plot the timeseries of mean density\n", + "fig = px.line(\n", + " mean_values, x=mean_values.index, y='value',\n", + " title=f'Mean Density - {site_id}',\n", + " labels={'value': 'Density', 'date': 'Date'}\n", + ")\n", + "\n", + "fig.update_layout(\n", + " template='plotly_dark'\n", + ")\n", + "\n", + "# Show the plot\n", + "fig.show()\n", + "\n", + "# alternative matplotlib code\n", + "# mean_values[\"value\"].plot()\n", + "# plt.title('Mean Density by Date')\n", + "# plt.xlabel('Date')\n", + "# plt.ylabel('Mean Density')\n", + "# plt.gcf().autofmt_xdate()\n", + "# plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "hoverinfo": "y+name", + "hovertemplate": "date=%{x}
value=%{y}", + "legendgroup": "", + "marker": { + "color": "#636efa" + }, + "name": "", + "notched": true, + "offsetgroup": "", + "orientation": "v", + "showlegend": false, + "type": "box", + "x": [ + "2019-12-18", + "2019-12-18", + "2019-12-18", + "2019-12-18", + "2019-12-18", + "2019-12-18", + "2019-12-18", + "2019-12-18", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-09", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-23", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-01-30", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-06", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-13", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-19", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-02-27", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-05", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12", + "2020-03-12" + ], + "x0": " ", + "xaxis": "x", + "y": [ + 268, + 377, + 228, + 243, + 268, + 377, + 228, + 243, + 60, + 65, + 116, + 133, + 198, + 138, + 163, + 273, + 265, + 60, + 65, + 116, + 133, + 198, + 138, + 163, + 273, + 265, + 121, + 175.5, + 204.66666666666666, + 221, + 254.5, + 282.5, + 312, + 299.5, + 315.5, + 317.6666666666667, + 292, + 121, + 175.5, + 204.66666666666663, + 221, + 254.5, + 282.5, + 312, + 299.5, + 315.5, + 317.6666666666667, + 292, + 130, + 138.5, + 133, + 157, + 202.5, + 262.5, + 222.5, + 315, + 336, + 330, + 302.5, + 302.5, + 130, + 138.5, + 133, + 157, + 202.5, + 262.5, + 222.5, + 315, + 336, + 330, + 302.5, + 302.5, + 89, + 118, + 175.33333333333334, + 189.5, + 213, + 264.5, + 293.5, + 316, + 339.5, + 342.5, + 322, + 311, + 330, + 89, + 118, + 175.33333333333334, + 189.5, + 213, + 264.5, + 293.5, + 316, + 339.5, + 342.5, + 322, + 311, + 330, + 146.5, + 152, + 207, + 239.5, + 253.5, + 298, + 329, + 346.5, + 346.5, + 344.5, + 332.5, + 318.5, + null, + 146.5, + 152, + 207, + 239.5, + 253.5, + 298, + 329, + 346.5, + 346.5, + 344.5, + 332.5, + 318.5, + null, + 114.66666666666667, + 161, + 191.5, + 230.5, + 257.5, + 274, + 316, + 350.5, + 362, + 370, + 336, + 333.5, + null, + 114.66666666666669, + 161, + 191.5, + 230.5, + 257.5, + 274, + 316, + 350.5, + 362, + 370, + 336, + 333.5, + null, + 158, + 192, + 233.5, + 267, + 284, + 328, + 356.5, + 368.5, + 373.5, + 344.5, + 340, + null, + 158, + 192, + 233.5, + 267, + 284, + 328, + 356.5, + 368.5, + 373.5, + 344.5, + 340, + null, + 233.5, + 227.5, + 274, + 296, + 327, + 353, + 369.5, + 379, + 378.5, + 338, + 343.3333333333333, + 233.5, + 227.5, + 274, + 296, + 327, + 353, + 369.5, + 379, + 378.5, + 338, + 343.3333333333333, + 204.5, + 271, + 265, + 313.5, + 312, + 347, + 374.5, + 394, + 381.5, + 352.5, + 348, + 204.5, + 271, + 265, + 313.5, + 312, + 347, + 374.5, + 394, + 381.5, + 352.5, + 348 + ], + "y0": " ", + "yaxis": "y" + } + ], + "layout": { + "boxmode": "group", + "legend": { + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#f2f5fa" + }, + "error_y": { + "color": "#f2f5fa" + }, + "marker": { + "line": { + "color": "rgb(17,17,17)", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "rgb(17,17,17)", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#A2B1C6", + "gridcolor": "#506784", + "linecolor": "#506784", + "minorgridcolor": "#506784", + "startlinecolor": "#A2B1C6" + }, + "baxis": { + "endlinecolor": "#A2B1C6", + "gridcolor": "#506784", + "linecolor": "#506784", + "minorgridcolor": "#506784", + "startlinecolor": "#A2B1C6" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "marker": { + "line": { + "color": "#283442" + } + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "line": { + "color": "#283442" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#506784" + }, + "line": { + "color": "rgb(17,17,17)" + } + }, + "header": { + "fill": { + "color": "#2a3f5f" + }, + "line": { + "color": "rgb(17,17,17)" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#f2f5fa", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#f2f5fa" + }, + "geo": { + "bgcolor": "rgb(17,17,17)", + "lakecolor": "rgb(17,17,17)", + "landcolor": "rgb(17,17,17)", + "showlakes": true, + "showland": true, + "subunitcolor": "#506784" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "dark" + }, + "paper_bgcolor": "rgb(17,17,17)", + "plot_bgcolor": "rgb(17,17,17)", + "polar": { + "angularaxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + }, + "bgcolor": "rgb(17,17,17)", + "radialaxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "rgb(17,17,17)", + "gridcolor": "#506784", + "gridwidth": 2, + "linecolor": "#506784", + "showbackground": true, + "ticks": "", + "zerolinecolor": "#C8D4E3" + }, + "yaxis": { + "backgroundcolor": "rgb(17,17,17)", + "gridcolor": "#506784", + "gridwidth": 2, + "linecolor": "#506784", + "showbackground": true, + "ticks": "", + "zerolinecolor": "#C8D4E3" + }, + "zaxis": { + "backgroundcolor": "rgb(17,17,17)", + "gridcolor": "#506784", + "gridwidth": 2, + "linecolor": "#506784", + "showbackground": true, + "ticks": "", + "zerolinecolor": "#C8D4E3" + } + }, + "shapedefaults": { + "line": { + "color": "#f2f5fa" + } + }, + "sliderdefaults": { + "bgcolor": "#C8D4E3", + "bordercolor": "rgb(17,17,17)", + "borderwidth": 1, + "tickwidth": 0 + }, + "ternary": { + "aaxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + }, + "baxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + }, + "bgcolor": "rgb(17,17,17)", + "caxis": { + "gridcolor": "#506784", + "linecolor": "#506784", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "updatemenudefaults": { + "bgcolor": "#506784", + "borderwidth": 0 + }, + "xaxis": { + "automargin": true, + "gridcolor": "#283442", + "linecolor": "#506784", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "#283442", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "#283442", + "linecolor": "#506784", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "#283442", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Pit Density by Date" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "title": { + "text": "date" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "value" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Show more detail by using a box plot\n", + "# make sure values are floats\n", + "df[\"value\"] = df[\"value\"].astype(float)\n", + "\n", + "# make Make a box plot\n", + "fig = px.box(df, x='date', y='value', notched=True, title='Pit Density by Date')\n", + "fig.update_traces(hoverinfo='y+name')\n", + "fig.update_layout(template='plotly_dark')\n", + "\n", + "# alternative matplotlib code\n", + "# df.boxplot(by='date', column='value')\n", + "# plt.title('Density Distribution for Banner by Date')\n", + "# plt.suptitle('') # Suppress the automatic title\n", + "# plt.xlabel('Date')\n", + "# plt.ylabel('Value')\n", + "# plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/requirements.txt b/docs/requirements.txt index 5b40a72..956431b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,10 @@ -nbsphinx==0.8.5 +nbsphinx==0.9.4 sphinx-gallery==0.9.0 nbconvert>=6.4.3,<6.5.0 -sphinx==4.0.2 +sphinx>=7.1,<7.4 pandoc==1.0.2 ipython>7.0,<9.0 sphinxcontrib-apidoc==0.3.0 MarkupSafe<2.1.0 +plotly==5.22.0 +jupyterlab==4.2.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b6a4ac1..93dba6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ geoalchemy2>=0.6,<1.0 geopandas>=0.7,<1.0 psycopg2-binary>=2.9.0,<2.10.0 rasterio>=1.1.5 -SQLAlchemy < 2.0.0 +SQLAlchemy >= 2.0.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index ca79474..9f6f8a6 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,13 +1,11 @@ --r docs/requirements.txt -r requirements.txt -pip>=22,<23 +pip==23.3 bump2version==0.5.11 wheel==0.38.1 watchdog==0.9.0 flake8==3.7.8 -coverage==4.5.4 +tox==3.14.0 +coverage==5.5 twine==1.14.0 -pytest==6.2.3 -pytest-runner==5.1 -jupyterlab==2.2.10 -matplotlib==3.2.2 +pytest==6.2.4 +pytest-cov==2.12.1 \ No newline at end of file diff --git a/setup.py b/setup.py index f737d41..746c130 100644 --- a/setup.py +++ b/setup.py @@ -13,23 +13,18 @@ with open('requirements.txt') as req: requirements = req.read().split('\n') -with open('requirements_dev.txt') as req: - # Ignore the -r on the two lines - setup_requirements = req.read().split('\n')[2:] - -setup_requirements += requirements test_requirements = ['pytest>=3'] + requirements setup( author="Micah Johnson", - python_requires='>=3.6', + python_requires='>=3.8', classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'Intended Audience :: Developers', 'Natural Language :: English', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10' ], description="SQL Database software for SnowEx data", entry_points={ @@ -44,7 +39,6 @@ keywords='snowexsql', name='snowexsql', packages=find_packages(include=['snowexsql', 'snowexsql.*']), - setup_requires=setup_requirements, test_suite='tests', tests_require=test_requirements, url='https://github.com/SnowEx/snowexsql', diff --git a/snowexsql/api.py b/snowexsql/api.py new file mode 100644 index 0000000..a44e656 --- /dev/null +++ b/snowexsql/api.py @@ -0,0 +1,328 @@ +import logging +from contextlib import contextmanager +from sqlalchemy.sql import func +import geopandas as gpd +from shapely.geometry import box +from geoalchemy2.shape import from_shape, to_shape +import geoalchemy2.functions as gfunc +from geoalchemy2.types import Raster + +from snowexsql.db import get_db +from snowexsql.data import SiteData, PointData, LayerData, ImageData +from snowexsql.conversions import query_to_geopandas, raster_to_rasterio + + +LOG = logging.getLogger(__name__) +DB_NAME = 'snow:hackweek@db.snowexdata.org/snowex' + +# TODO: +# * Possible enums +# * filtering based on dates +# * implement 'like' or 'contains' method + + +class LargeQueryCheckException(RuntimeError): + pass + + +@contextmanager +def db_session(db_name): + # use default_name + db_name = db_name or DB_NAME + engine, session = get_db(db_name) + yield session, engine + session.close() + + +def get_points(): + # Lets grab a single row from the points table + with db_session(DB_NAME) as session: + qry = session.query(PointData).limit(1) + # Execute that query! + result = qry.all() + + +class BaseDataset: + MODEL = None + # Use this database name + DB_NAME = DB_NAME + + ALLOWED_QRY_KWARGS = [ + "site_name", "site_id", "date", "instrument", "observers", "type", + "utm_zone", "date_greater_equal", "date_less_equal" + ] + SPECIAL_KWARGS = ["limit"] + # Default max record count + MAX_RECORD_COUNT = 1000 + + @staticmethod + def build_box(xmin, ymin, xmax, ymax, crs): + # build a geopandas box + return gpd.GeoDataFrame( + geometry=[box(xmin, ymin, xmax, ymax)] + ).set_crs(crs) + + @classmethod + def _check_size(cls, qry, kwargs): + # Safe guard against accidental giant requests + count = qry.count() + if count > cls.MAX_RECORD_COUNT and "limit" not in kwargs: + raise LargeQueryCheckException( + f"Query will return {count} number of records," + f" but we have a default max of {cls.MAX_RECORD_COUNT}." + f" If you want to proceed, set the 'limit' filter" + f" to the desired number of records." + ) + + @classmethod + def extend_qry(cls, qry, check_size=True, **kwargs): + if cls.MODEL is None: + raise ValueError("You must use a class with a MODEL.") + + # use the default kwargs + for k, v in kwargs.items(): + # Handle special operations + if k in cls.ALLOWED_QRY_KWARGS: + # standard filtering using qry.filter + if isinstance(v, list): + filter_col = getattr(cls.MODEL, k) + if k == "date": + raise ValueError( + "We cannot search for a list of dates" + ) + elif "_equal" in k: + raise ValueError( + "We cannot compare greater_equal or less_equal" + " with a list" + ) + qry = qry.filter(filter_col.in_([v])) + LOG.debug( + f"Filtering {k} to value {v}" + ) + else: + # Filter boundary + if "_greater_equal" in k: + key = k.split("_greater_equal")[0] + filter_col = getattr(cls.MODEL, key) + qry = qry.filter(filter_col >= v) + elif "_less_equal" in k: + key = k.split("_less_equal")[0] + filter_col = getattr(cls.MODEL, key) + qry = qry.filter(filter_col <= v) + # Filter to exact value + else: + filter_col = getattr(cls.MODEL, k) + qry = qry.filter(filter_col == v) + LOG.debug( + f"Filtering {k} to list {v}" + ) + + # to avoid limit before filter + elif k in cls.SPECIAL_KWARGS: + if k == "limit": + qry = qry.limit(v) + + else: + # Error out for not-allowed kwargs + raise ValueError(f"{k} is not an allowed filter") + + if check_size: + cls._check_size(qry, kwargs) + + return qry + + @property + def all_site_names(self): + """ + Return all types of the data + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(self.MODEL.site_name).distinct() + result = qry.all() + return result + + @property + def all_types(self): + """ + Return all types of the data + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(self.MODEL.type).distinct() + result = qry.all() + return result + + @property + def all_dates(self): + """ + Return all distinct dates in the data + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(self.MODEL.date).distinct() + result = qry.all() + return result + + @property + def all_observers(self): + """ + Return all distinct observers in the data + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(self.MODEL.observers).distinct() + result = qry.all() + return result + + @property + def all_instruments(self): + """ + Return all distinct instruments in the data + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(self.MODEL.instrument).distinct() + result = qry.all() + return result + + +class PointMeasurements(BaseDataset): + """ + API class for access to PointData + """ + MODEL = PointData + + @classmethod + def from_filter(cls, **kwargs): + """ + Get data for the class by filtering by allowed arguments. The allowed + filters are cls.ALLOWED_QRY_KWARGS. + """ + with db_session(cls.DB_NAME) as (session, engine): + try: + qry = session.query(cls.MODEL) + qry = cls.extend_qry(qry, **kwargs) + df = query_to_geopandas(qry, engine) + except Exception as e: + session.close() + LOG.error("Failed query for PointData") + raise e + + return df + + @classmethod + def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs): + """ + Get data for the class within a specific shapefile or + within a point and a known buffer + Args: + shp: shapely geometry in which to filter + pt: shapely point that will have a buffer applied in order + to find search area + buffer: in same units as point + crs: integer crs to use + kwargs: for more filtering or limiting (cls.ALLOWED_QRY_KWARGS) + Returns: Geopandas dataframe of results + + """ + if shp is None and pt is None: + raise ValueError( + "Inputs must be a shape description or a point and buffer" + ) + if (pt is not None and buffer is None) or \ + (buffer is not None and pt is None): + raise ValueError("pt and buffer must be given together") + with db_session(cls.DB_NAME) as (session, engine): + try: + if shp is not None: + qry = session.query(cls.MODEL) + qry = qry.filter( + func.ST_Within( + cls.MODEL.geom, from_shape(shp, srid=crs) + ) + ) + qry = cls.extend_qry(qry, check_size=True, **kwargs) + df = query_to_geopandas(qry, engine) + else: + qry_pt = from_shape(pt) + qry = session.query( + gfunc.ST_SetSRID( + func.ST_Buffer(qry_pt, buffer), crs + ) + ) + + buffered_pt = qry.all()[0][0] + qry = session.query(cls.MODEL) + qry = qry.filter(func.ST_Within(cls.MODEL.geom, buffered_pt)) + qry = cls.extend_qry(qry, check_size=True, **kwargs) + df = query_to_geopandas(qry, engine) + except Exception as e: + session.close() + raise e + + return df + + +class LayerMeasurements(PointMeasurements): + """ + API class for access to LayerData + """ + MODEL = LayerData + ALLOWED_QRY_KWARGS = [ + "site_name", "site_id", "date", "instrument", "observers", "type", + "utm_zone", "pit_id", "date_greater_equal", "date_less_equal" + ] + # TODO: layer analysis methods? + + +class RasterMeasurements(BaseDataset): + MODEL = ImageData + + @classmethod + def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs): + if shp is None and pt is None: + raise ValueError( + "We need a shape description or a point and buffer") + if (pt is not None and buffer is None) or ( + buffer is not None and pt is None): + raise ValueError("pt and buffer must be given together") + with db_session(cls.DB_NAME) as (session, engine): + try: + # Grab the rasters, union them and convert them as tiff when done + q = session.query( + func.ST_AsTiff( + func.ST_Union(ImageData.raster, type_=Raster) + ) + ) + # Query upfront except for the limit + limit = kwargs.get("limit") + if limit: + kwargs.pop("limit") + q = cls.extend_qry(q, check_size=False, **kwargs) + if shp: + q = q.filter( + gfunc.ST_Intersects( + ImageData.raster, + from_shape(shp, srid=crs) + ) + ) + else: + qry_pt = from_shape(pt) + qry = session.query( + gfunc.ST_SetSRID( + func.ST_Buffer(qry_pt, buffer), crs + ) + ) + buffered_pt = qry.all()[0][0] + # And grab rasters touching the circle + q = q.filter(gfunc.ST_Intersects(ImageData.raster, buffered_pt)) + # Execute the query + # Check the query size or limit the query + if limit: + q = cls.extend_qry(q, limit=limit) + else: + cls._check_size(qry, kwargs) + rasters = q.all() + # Get the rasterio object of the raster + dataset = raster_to_rasterio(session, rasters)[0] + return dataset + + except Exception as e: + session.close() + raise e diff --git a/snowexsql/conversions.py b/snowexsql/conversions.py index 1802515..0db6901 100644 --- a/snowexsql/conversions.py +++ b/snowexsql/conversions.py @@ -44,8 +44,8 @@ def points_to_geopandas(results): def query_to_geopandas(query, engine, **kwargs): """ - Convert a GeoAlchemy2 Query meant for postgis to a geopandas dataframe. Requires that a geometry column is - included + Convert a GeoAlchemy2 Query meant for postgis to a geopandas dataframe. + Requires that a geometry column is included Args: query: GeoAlchemy2.Query Object diff --git a/snowexsql/db.py b/snowexsql/db.py index 607469d..009737c 100644 --- a/snowexsql/db.py +++ b/snowexsql/db.py @@ -60,7 +60,7 @@ def get_db(db_str, credentials=None, return_metadata=False): "options": "-c timezone=UTC"}) Session = sessionmaker(bind=engine) - metadata = MetaData(bind=engine) + metadata = MetaData() session = Session(expire_on_commit=False) if return_metadata: diff --git a/tests/map.html b/tests/map.html new file mode 100644 index 0000000..f85b583 --- /dev/null +++ b/tests/map.html @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + \ No newline at end of file diff --git a/tests/scratch.py b/tests/scratch.py new file mode 100644 index 0000000..7c7a40e --- /dev/null +++ b/tests/scratch.py @@ -0,0 +1,35 @@ +from metloom.pointdata import SnotelPointData +from os.path import join, dirname + +import pytest +import geopandas as gpd +from datetime import date + +from snowexsql.api import PointMeasurements + + +def test_stuff(): + sntl_point = SnotelPointData("622:CO:SNTL", "dummy name") + geom = sntl_point.metadata + geom = gpd.GeoSeries(geom).set_crs(4326).to_crs(26912).geometry.values[0] + + shp1 = gpd.GeoSeries( + sntl_point.metadata + ).set_crs(4326).buffer(.1).total_bounds + bx = PointMeasurements.build_box( + *list(shp1), + 4326 + ) + bx = bx.to_crs(26912) + bx.explore().save("map.html") + df = PointMeasurements.from_area( + shp=bx.geometry.iloc[0], limit=30 + ) + + df = PointMeasurements.from_area( + pt=geom, buffer=10000, instrument="magnaprobe", limit=250 + ) + # df = PointMeasurements.from_filter( + # instrument="magnaprobe", limit=20 + # ) + print(df) diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..46fa128 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,250 @@ +from os.path import join, dirname +import geopandas as gpd +import numpy as np +import pytest +from datetime import date + +from snowexsql.api import ( + PointMeasurements, LargeQueryCheckException, LayerMeasurements +) +from snowexsql.db import get_db, initialize + + +@pytest.fixture(scope="session") +def data_dir(): + return join(dirname(__file__), 'data') + + +@pytest.fixture(scope="session") +def creds(data_dir): + return join(dirname(__file__), 'credentials.json') + + +@pytest.fixture(scope="session") +def db_url(): + return 'localhost/test' + + +class DBConnection: + """ + Base class for connecting to the test database and overwiting the URL + so that we stay connected to our local testing DB + """ + CLZ = PointMeasurements + + @pytest.fixture(scope="class") + def db(self, creds, db_url): + engine, session, metadata = get_db( + db_url, credentials=creds, return_metadata=True) + + initialize(engine) + yield engine + # cleanup + session.flush() + session.rollback() + metadata.drop_all(bind=engine) + session.close() + + @pytest.fixture(scope="class") + def clz(self, db, db_url): + """ + Extend the class and overwrite the database name + """ + url = db.url + class Extended(self.CLZ): + DB_NAME = f"{url.username}:{url.password}@{url.host}/{url.database}" + + yield Extended + + +def unsorted_list_tuple_compare(l1, l2): + # turn lists into sets, but get rid of any Nones + l1 = set([l[0] for l in l1 if l[0] is not None]) + l2 = set([l[0] for l in l2 if l[0] is not None]) + # compare the sets + return l1 == l2 + + +class TestPointMeasurements(DBConnection): + """ + Test the Point Measurement class + """ + CLZ = PointMeasurements + + def test_all_types(self, clz): + result = clz().all_types + assert unsorted_list_tuple_compare( + result, + [] + ) + + def test_all_site_names(self, clz): + result = clz().all_site_names + assert unsorted_list_tuple_compare( + result, [] + ) + + def test_all_dates(self, clz): + result = clz().all_dates + assert len(result) == 0 + + def test_all_observers(self, clz): + result = clz().all_observers + assert unsorted_list_tuple_compare( + result, [] + ) + + def test_all_instruments(self, clz): + result = clz().all_instruments + assert unsorted_list_tuple_compare( + result, [] + ) + + @pytest.mark.parametrize( + "kwargs, expected_length, mean_value", [ + ({ + "date": date(2020, 5, 28), + "instrument": 'camera' + }, 0, np.nan), + ({"instrument": "magnaprobe", "limit": 10}, 0, np.nan), # limit works + ({ + "date": date(2020, 5, 28), + "instrument": 'pit ruler' + }, 0, np.nan), + ({ + "date_less_equal": date(2019, 10, 1), + }, 0, np.nan), + ({ + "date_greater_equal": date(2020, 6, 7), + }, 0, np.nan), + ] + ) + def test_from_filter(self, clz, kwargs, expected_length, mean_value): + result = clz.from_filter(**kwargs) + assert len(result) == expected_length + if expected_length > 0: + assert pytest.approx(result["value"].mean()) == mean_value + + @pytest.mark.parametrize( + "kwargs, expected_error", [ + ({"notakey": "value"}, ValueError), + # ({"instrument": "magnaprobe"}, LargeQueryCheckException), + ({"date": [date(2020, 5, 28), date(2019, 10, 3)]}, ValueError), + ] + ) + def test_from_filter_fails(self, clz, kwargs, expected_error): + """ + Test failure on not-allowed key and too many returns + """ + with pytest.raises(expected_error): + clz.from_filter(**kwargs) + + def test_from_area(self, clz): + shp = gpd.points_from_xy( + [743766.4794971556], [4321444.154620216], crs="epsg:26912" + ).buffer(10)[0] + result = clz.from_area( + shp=shp, + date=date(2019, 10, 30) + ) + assert len(result) == 0 + + def test_from_area_point(self, clz): + pts = gpd.points_from_xy([743766.4794971556], [4321444.154620216]) + crs = "26912" + result = clz.from_area( + pt=pts[0], buffer=10, crs=crs, + date=date(2019, 10, 30) + ) + assert len(result) == 0 + + +class TestLayerMeasurements(DBConnection): + """ + Test the Layer Measurement class + """ + CLZ = LayerMeasurements + + def test_all_types(self, clz): + result = clz().all_types + assert result == [] + + def test_all_site_names(self, clz): + result = clz().all_site_names + assert result == [] + + def test_all_dates(self, clz): + result = clz().all_dates + assert len(result) == 0 + + def test_all_observers(self, clz): + result = clz().all_observers + assert unsorted_list_tuple_compare(result, []) + + def test_all_instruments(self, clz): + result = clz().all_instruments + assert unsorted_list_tuple_compare(result, []) + + @pytest.mark.parametrize( + "kwargs, expected_length, mean_value", [ + ({ + "date": date(2020, 3, 12), "type": "density", + "pit_id": "COERIB_20200312_0938" + }, 0, np.nan), # filter to 1 pit + ({"instrument": "IRIS", "limit": 10}, 0, np.nan), # limit works + ({ + "date": date(2020, 5, 28), + "instrument": 'IRIS' + }, 0, np.nan), # nothing returned + ({ + "date_less_equal": date(2019, 12, 15), + "type": 'density' + }, 0, np.nan), + ({ + "date_greater_equal": date(2020, 5, 13), + "type": 'density' + }, 0, np.nan), + ] + ) + def test_from_filter(self, clz, kwargs, expected_length, mean_value): + result = clz.from_filter(**kwargs) + assert len(result) == expected_length + if expected_length > 0: + assert pytest.approx( + result["value"].astype("float").mean() + ) == mean_value + + @pytest.mark.parametrize( + "kwargs, expected_error", [ + ({"notakey": "value"}, ValueError), + # ({"date": date(2020, 3, 12)}, LargeQueryCheckException), + ({"date": [date(2020, 5, 28), date(2019, 10, 3)]}, ValueError), + ] + ) + def test_from_filter_fails(self, clz, kwargs, expected_error): + """ + Test failure on not-allowed key and too many returns + """ + with pytest.raises(expected_error): + clz.from_filter(**kwargs) + + def test_from_area(self, clz): + df = gpd.GeoDataFrame( + geometry=gpd.points_from_xy( + [743766.4794971556], [4321444.154620216], crs="epsg:26912" + ).buffer(1000.0) + ).set_crs("epsg:26912") + result = clz.from_area( + type="density", + shp=df.iloc[0].geometry, + ) + assert len(result) == 0 + + def test_from_area_point(self, clz): + pts = gpd.points_from_xy([743766.4794971556], [4321444.154620216]) + crs = "26912" + result = clz.from_area( + pt=pts[0], buffer=1000, crs=crs, + type="density", + ) + assert len(result) == 0 diff --git a/tests/test_db.py b/tests/test_db.py index 223b6b5..6b38676 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -37,6 +37,8 @@ def setup_class(self): """ super().setup_class() site_fname = join(self.data_dir, 'site_details.csv') + # only reflect the tables we will use + self.metadata.reflect(self.engine, only=['points', 'layers']) def test_point_structure(self): """