Merge pull request #112 from DHI-GRAS/wtd

WIP on docs

.travis.yml

@@ -1,6 +1,9 @@
 language: python
 
+dist: xenial
+
 python:
+  - "3.7"
   - "3.6"
 
 env:

README.rst

@@ -4,9 +4,10 @@
 Terracotta
 ==========
 
-`Try the demo <https://satlas.dk>`__ \| 
-`Read the docs <https://terracotta-python.readthedocs.org>`__ \|
-`Explore the API <https://2truhxo59g.execute-api.eu-central-1.amazonaws.com/production/apidoc>`__
+`Try the demo <https://terracotta-python.readthedocs.io/en/latest/preview-app.html>`__ \|
+`Read the docs <https://terracotta-python.readthedocs.io/en/latest>`__ \|
+`Explore the API <https://2truhxo59g.execute-api.eu-central-1.amazonaws.com/production/apidoc>`__ \|
+`Satlas, powered by Terracotta <http://satlas.dk>`__
 
 A light-weight, versatile XYZ tile server, built with Flask and Rasterio
 :earth_africa:

docs/api.rst

@@ -1,43 +1,44 @@
-.. _api:
-
 Python API
 ==========
 
-Top-level interface
--------------------
-
-.. autofunction:: terracotta.get_driver
+Get and set runtime settings
+----------------------------
 
 .. autofunction:: terracotta.get_settings
 
 .. autofunction:: terracotta.update_settings
 
-Drivers
--------
+.. _drivers:
 
-Common interface
-++++++++++++++++
-
-.. autoclass:: terracotta.drivers.base.Driver
-   :members:
+Get a driver instance
+---------------------
 
-Available drivers
-+++++++++++++++++
+.. autofunction:: terracotta.get_driver
 
-SQLite
-~~~~~~
+SQLite driver
+-------------
 
 .. autoclass:: terracotta.drivers.sqlite.SQLiteDriver
-   :members: __init__
+   :members:
+   :undoc-members:
+   :special-members: __init__
+   :inherited-members:
 
-Remote SQLite on S3
-~~~~~~~~~~~~~~~~~~~
+Remote SQLite driver
+--------------------
 
 .. autoclass:: terracotta.drivers.sqlite_remote.RemoteSQLiteDriver
-   :members: __init__
+   :members:
+   :undoc-members:
+   :special-members: __init__
+   :inherited-members:
+   :exclude-members: delete, insert, create
 
-MySQL
-~~~~~
+MySQL driver
+------------
 
 .. autoclass:: terracotta.drivers.mysql.MySQLDriver
-   :members: __init__
+   :members:
+   :undoc-members:
+   :special-members: __init__
+   :inherited-members:

docs/cli-commands/optimize-rasters.rst

@@ -1,2 +1,2 @@
 .. click:: terracotta.scripts.cli:optimize_rasters
-  :prog: terracotta optimize-rasters
+  :prog: terracotta optimize-rasters

docs/cli.rst

@@ -5,7 +5,7 @@ Command-line interface
 
 .. toctree::
    :maxdepth: 1
-   :caption: Available commands
+   :caption: Available commands:
 
    cli-commands/main
    cli-commands/optimize-rasters

docs/conf.py

@@ -18,14 +18,15 @@
 
 
 # -- Project information -----------------------------------------------------
+import re
 from terracotta import __version__
 
 project = 'Terracotta'
 copyright = '2018, the Terracotta contributors'
 author = 'Dion Häfner, Philip Graae'
 
 # The short X.Y version
-version = __version__
+version = re.match(r'(\d+\.\d+\.\d+)', __version__).group(1)
 # The full version, including alpha/beta/rc tags
 release = __version__
 
@@ -40,16 +41,14 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinx.ext.intersphinx',
     'sphinx.ext.autodoc',
     'sphinx.ext.napoleon',
     'sphinx.ext.viewcode',
     'sphinx_autodoc_typehints',
     'sphinx_click.ext'
 ]
 
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
@@ -74,6 +73,34 @@
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'friendly'
 
+# -- Dirty HAXX to compile and serve preview app -----------------------------
+
+preview_hostname = 'https://2truhxo59g.execute-api.eu-central-1.amazonaws.com/production'
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates', '../terracotta/client/templates']
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static', '../terracotta/client/static']
+
+html_additional_pages = {
+    'preview-app': 'app.html',
+}
+
+# Inject Jinja variables defined by Flask
+html_context = {
+    'hostname': preview_hostname,
+    'url_for': lambda _, filename: f'_static/{filename}'
+}
+
+# -- Extension settings --------------------------------------------------------
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+    'numpy': ('http://docs.scipy.org/doc/numpy/', None),
+}
 
 # -- Options for HTML output -------------------------------------------------
 
@@ -90,9 +117,11 @@
     'description': 'A light-weight, versatile XYZ tile server built with Flask and Rasterio',
     'code_font_family': "'Roboto Mono', 'Consolas', 'Menlo', 'Deja Vu Sans Mono', "
                         "'Bitstream Vera Sans Mono', monospace",
-    'font_family': "'Lato', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro', sans-serif",
-    'head_font_family': "'Lato', 'Garamond', 'Georgia', sans-serif",
+    'font_family': "'Lato', Arial, sans-serif",
+    'head_font_family': "'Lato', Arial, sans-serif",
     'body_text': '#000',
+    'sidebar_header': '#4B4032',
+    'sidebar_text': '#49443E',
     'github_banner': 'true',
     'github_user': 'DHI-GRAS',
     'github_repo': 'terracotta',
@@ -106,12 +135,6 @@
 def setup(app):
     app.add_stylesheet('https://fonts.googleapis.com/css?family=Lato|Roboto+Mono')
 
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
 #

docs/example-ingestion-script.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import glob
+
+import tqdm
+import boto3
+s3 = boto3.resource('s3')
+
+import terracotta as tc
+
+# settings
+DB_NAME = 'terracotta.sqlite'
+RASTER_GLOB = r'/path/to/rasters/*.tif'
+RASTER_NAME_PATTERN = r'(?P<sensor>\w{2})_(?P<tile>\w{5})_(?P<date>\d{8})_(?P<band>\w+).tif'
+KEYS = ('sensor', 'tile', 'date', 'band')
+KEY_DESCRIPTIONS = {
+    'sensor': 'Sensor short name',
+    'tile': 'Sentinel-2 tile ID',
+    'date': 'Sensing date',
+    'band': 'Band or index name'
+}
+S3_BUCKET = 'tc-testdata'
+S3_RASTER_FOLDER = 'rasters'
+S3_PATH = f's3://{S3_BUCKET}/{S3_RASTER_FOLDER}'
+
+driver = tc.get_driver(DB_NAME)
+
+# create an empty database if it doesn't exist
+if not os.path.isfile(DB_NAME):
+    driver.create(KEYS, KEY_DESCRIPTIONS)
+
+# sanity check
+assert driver.key_names == KEYS
+
+available_datasets = driver.get_datasets()
+raster_files = list(glob.glob(RASTER_GLOB))
+pbar = tqdm.tqdm(raster_files)
+
+for raster_path in pbar:
+    pbar.set_postfix(file=raster_path)
+
+    raster_filename = os.path.basename(raster_path)
+
+    # extract keys from filename
+    match = re.match(RASTER_NAME_PATTERN, raster_filename)
+    if match is None:
+        raise ValueError(f'Input file {raster_filename} does not match raster pattern')
+
+    keys = match.groups()
+
+    # skip already processed data
+    if keys in available_datasets:
+            continue
+
+    with driver.connect():
+        # since the rasters will be served from S3, we need to pass the correct remote path
+        driver.insert(keys, raster_path, override_path=f'{S3_PATH}/{raster_filename}')
+        s3.meta.client.upload_file(raster_path, S3_BUCKET,
+                                   f'{S3_RASTER_FOLDER}/{raster_filename}')
+
+# upload database to S3
+s3.meta.client.upload_file(DB_NAME, S3_BUCKET, DB_NAME)