From 03aa8d65a11eb9f15219781e22fbc6730ff0da61 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 11 Aug 2024 15:41:00 +0200 Subject: [PATCH] Added documentation using Sphinx and Read the Docs --- .gitignore | 1 + .readthedocs.yml | 29 ++++++++ CHANGES.md | 1 + README.md | 28 ++++---- doc/Makefile | 20 ++++++ doc/_static/.gitkeep | 0 doc/_templates/.gitkeep | 0 {docs => doc}/backlog.md | 2 +- doc/cdc/index.md | 30 ++++++++ {docs => doc/cdc}/mongodb.md | 1 + doc/changes.md | 1 + doc/conf.py | 130 +++++++++++++++++++++++++++++++++++ doc/decode.md | 27 ++++++++ doc/development.md | 26 +++++++ doc/index.md | 78 +++++++++++++++++++++ doc/make.bat | 35 ++++++++++ doc/prior-art.md | 6 ++ doc/readme.md | 1 + pyproject.toml | 25 ++++++- 19 files changed, 424 insertions(+), 17 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 doc/Makefile create mode 100644 doc/_static/.gitkeep create mode 100644 doc/_templates/.gitkeep rename {docs => doc}/backlog.md (92%) create mode 100644 doc/cdc/index.md rename {docs => doc/cdc}/mongodb.md (99%) create mode 120000 doc/changes.md create mode 100644 doc/conf.py create mode 100644 doc/decode.md create mode 100644 doc/development.md create mode 100644 doc/index.md create mode 100644 doc/make.bat create mode 100644 doc/prior-art.md create mode 120000 doc/readme.md diff --git a/.gitignore b/.gitignore index d3123a0..8505ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /build /.coverage /coverage.xml +/doc/_build /dist /.idea /.venv* diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..512a0db --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,29 @@ +# .readthedocs.yml +# Read the Docs configuration file + +# Details +# - https://docs.readthedocs.io/en/stable/config-file/v2.html + +# Required +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + +python: + install: + - method: pip + path: . + extra_requirements: + - doc + +sphinx: + configuration: doc/conf.py + builder: html + fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF +#formats: +# - pdf diff --git a/CHANGES.md b/CHANGES.md index b970185..5abd9a1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Changelog ## Unreleased +- Added documentation using Sphinx and Read the Docs ## 2024/08/05 v0.0.3 - Added transformer for AWS DMS to CrateDB SQL diff --git a/README.md b/README.md index 0bfee94..eba78af 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,13 @@ Data decoding, encoding, conversion, and translation utilities. > > -- https://en.wikipedia.org/wiki/Codec -## Details -A collection of reusable utilities with minimal dependencies for transcoding -purposes, mostly collected from other projects like [Kotori] and [LorryStream], -in order to provide them per standalone package for broader use cases. +## What's Inside +- **Decoders:** A collection of reusable utilities with minimal dependencies for + transcoding purposes, mostly collected from other projects like + [Kotori](https://kotori.readthedocs.io/) and [LorryStream](https://lorrystream.readthedocs.io/), + in order to provide them per standalone package for broader use cases. + +- Transformers for [Change Data Capture (CDC)] messages to SQL statements. ## Installation The package is available from [PyPI] at [commons-codec]. @@ -43,14 +46,13 @@ and explore the source code or its [examples]. Kudos to the authors of all the many software components this library is vendoring and building upon. -### Prior Art -- [core-cdc] by Alejandro Cora González. Cheers! +### Similar Projects +See [prior art]. ### Contributing The `commons-codec` package is an open source project, and is -[managed on GitHub](https://github.com/daq-tools/commons-codec). -The project is still in its infancy, and we appreciate contributions -of any kind. +[managed on GitHub]. The project is still in its infancy, and +we appreciate contributions of any kind. ### Etymology The [Apache Commons Codec] library was the inspiration for the name. Otherwise, @@ -65,10 +67,10 @@ within the header sections of relevant files. [Apache Commons Codec]: https://commons.apache.org/proper/commons-codec/ +[Change Data Capture (CDC)]: https://en.wikipedia.org/wiki/Change_data_capture [commons-codec]: https://pypi.org/project/commons-codec/ -[core-cdc]: https://pypi.org/project/core-cdc/ -[documentation]: https://github.com/daq-tools/commons-codec/tree/main/docs +[documentation]: https://commons-codec.readthedocs.io/ [examples]: https://github.com/daq-tools/commons-codec/tree/main/examples -[Kotori]: https://github.com/daq-tools/kotori -[LorryStream]: https://github.com/daq-tools/lorrystream/ +[managed on GitHub]: https://github.com/daq-tools/commons-codec +[prior art]: https://commons-codec.readthedocs.io/prior-art.html [PyPI]: https://pypi.org/ diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/_static/.gitkeep b/doc/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/doc/_templates/.gitkeep b/doc/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/backlog.md b/doc/backlog.md similarity index 92% rename from docs/backlog.md rename to doc/backlog.md index 4ebf7c0..91b7814 100644 --- a/docs/backlog.md +++ b/doc/backlog.md @@ -8,7 +8,7 @@ - [x] Release v0.0.2 ## Iteration +2 -- [ ] Performance: Batching! +- [ ] Performance: Batching!? - [ ] MongoDB: Implement stream resumption using `start_after` - [ ] Feature: Filter by events, e.g. Ignore "delete" events? - [ ] Integration Testing the "example" programs? diff --git a/doc/cdc/index.md b/doc/cdc/index.md new file mode 100644 index 0000000..6208178 --- /dev/null +++ b/doc/cdc/index.md @@ -0,0 +1,30 @@ +# Change Data Capture (CDC) + +`commons-codec` includes CDC -> SQL transformer components for AWS DMS, +DynamoDB, and MongoDB. + +## DynamoDB +- Blog: [Replicating CDC Events from DynamoDB to CrateDB] +- Documentation: [DynamoDB CDC Relay for CrateDB] + +## MongoDB +- Introduction: [](project:#mongodb-cdc) +- Documentation: [MongoDB CDC Relay for CrateDB] + + +```{toctree} +:hidden: + +mongodb +``` + + +:::{note} +Please note relevant components are still in their infancy (beta), +and need further curation and improvements. +::: + + +[DynamoDB CDC Relay for CrateDB]: https://cratedb-toolkit.readthedocs.io/io/dynamodb/cdc.html +[MongoDB CDC Relay for CrateDB]: https://cratedb-toolkit.readthedocs.io/io/mongodb/cdc.html +[Replicating CDC Events from DynamoDB to CrateDB]: https://cratedb.com/blog/replicating-cdc-events-from-dynamodb-to-cratedb diff --git a/docs/mongodb.md b/doc/cdc/mongodb.md similarity index 99% rename from docs/mongodb.md rename to doc/cdc/mongodb.md index 30f731a..52c9898 100644 --- a/docs/mongodb.md +++ b/doc/cdc/mongodb.md @@ -1,3 +1,4 @@ +(mongodb-cdc)= # Relay MongoDB Change Stream into CrateDB ## About diff --git a/doc/changes.md b/doc/changes.md new file mode 120000 index 0000000..cf54708 --- /dev/null +++ b/doc/changes.md @@ -0,0 +1 @@ +../CHANGES.md \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..ad7af42 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,130 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +# ruff: noqa: ERA001 + +project = "commons-codec" +copyright = "2019-2024, The Panodata Developers" # noqa: A001 +author = "The Panodata Developers" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_parser", + "sphinx_copybutton", + "sphinx_design", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.ifconfig", + "sphinxcontrib.mermaid", + "sphinxext.opengraph", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. + +html_title = "commons-codec" + +html_theme_options = { + "sidebar_hide_name": False, + # https://github.com/pradyunsg/furo/blob/main/src/furo/assets/styles/variables/_colors.scss + # "light_logo": "logo-light.svg", + # "dark_logo": "logo-dark.svg", + # #CC3333 is persian red. + "light_css_variables": { + "color-brand-primary": "darkcyan", + "color-brand-content": "darkblue", + # "color-admonition-background": "orange", + }, + "dark_css_variables": { + "color-brand-primary": "darkcyan", + "color-brand-content": "lightblue", + # "color-admonition-background": "orange", + }, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + +html_show_sourcelink = True + + +# -- Intersphinx ---------------------------------------------------------- + +intersphinx_mapping = { + # "influxio": ("https://influxio.readthedocs.io/", None), +} +linkcheck_ignore = [] + +# Disable caching remote inventories completely. +# http://www.sphinx-doc.org/en/stable/ext/intersphinx.html#confval-intersphinx_cache_limit +# intersphinx_cache_limit = 0 + + +# -- Extension configuration ------------------------------------------------- + +sphinx_tabs_valid_builders = ["linkcheck"] +todo_include_todos = True + +# Configure sphinx-copybutton +copybutton_remove_prompts = True +copybutton_line_continuation_character = "\\" +copybutton_prompt_text = r">>> |\.\.\. |\$ |sh\$ |PS> |cr> |mysql> |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True + +# Configure sphinxext-opengraph +ogp_site_url = "https://commons-codec.readthedocs.io/" +ogp_enable_meta_description = True +# ogp_image = "http://example.org/image.png" +# ogp_description_length = 300 + + +# -- Options for MyST ------------------------------------------------- + +myst_heading_anchors = 3 +myst_enable_extensions = [ + "attrs_block", + "attrs_inline", + "colon_fence", + "deflist", + "fieldlist", + "html_admonition", + "html_image", + "linkify", + "replacements", + "strikethrough", + "substitution", + "tasklist", +] +myst_substitutions = {} diff --git a/doc/decode.md b/doc/decode.md new file mode 100644 index 0000000..f3f6c8a --- /dev/null +++ b/doc/decode.md @@ -0,0 +1,27 @@ +# Various Decoders + +`commons-codec` includes telemetry data decoders for individual popular sensor +appliances. + +## Sensor.Community + +- Air quality measurement appliances of [Sensor.Community]. + + Example: [Kotori Sensor.Community integration] + +- [Tasmota open source firmware for ESP devices]. + + Example: [Kotori Tasmota integration] + +- Devices connected to [The Things Stack (TTS)] / [The Things Network (TTN)]. + + Example: [Kotori TTN/TTS integration] + + +[Kotori Sensor.Community integration]: https://kotori.readthedocs.io/en/latest/integration/airrohr.html +[Kotori Tasmota integration]: https://kotori.readthedocs.io/en/latest/integration/tasmota.html +[Kotori TTN/TTS integration]: https://kotori.readthedocs.io/en/latest/integration/tts-ttn.html +[Sensor.Community]: https://sensor.community/ +[Tasmota open source firmware for ESP devices]: https://tasmota.github.io/ +[The Things Stack (TTS)]: https://www.thethingsindustries.com/docs/ +[The Things Network (TTN)]: https://www.thethingsnetwork.org/ diff --git a/doc/development.md b/doc/development.md new file mode 100644 index 0000000..6e03f50 --- /dev/null +++ b/doc/development.md @@ -0,0 +1,26 @@ +# Development Sandbox + +Acquire source code, install development sandbox, and invoke software tests. +```shell +git clone https://github.com/daq-tools/commons-codec +cd commons-codec +python3 -m venv .venv +source .venv/bin/activate +pip install --editable='.[all,develop,doc,test]' +poe check +``` + +Format code. +```shell +poe format +``` + +Run linter. +```shell +poe lint +``` + +Build documentation, with live-reloading. +```shell +poe docs-autobuild +``` diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..63d66b0 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,78 @@ +(index)= +# commons-codec + +[![CI][badge-tests]][project-tests] +[![Coverage Status][badge-coverage]][project-codecov] +[![Documentation][badge-documentation]][project-documentation] +[![License][badge-license]][project-license] +[![Downloads per month][badge-downloads-per-month]][project-downloads] + +[![Supported Python versions][badge-python-versions]][project-pypi] +[![Status][badge-status]][project-pypi] +[![Package version][badge-package-version]][project-pypi] + +» [Documentation][project-documentation] +| [Changelog][project-changelog] +| [PyPI][project-pypi] +| [Issues][project-issues] +| [Source code][project-repository] +| [License][project-license] + +» [CrateDB] +| [Kotori] +| [LorryStream] + +```{include} readme.md +:start-line: 12 +``` + + +```{toctree} +:maxdepth: 3 +:caption: Documentation +:hidden: + +cdc/index +decode +``` + +```{toctree} +:maxdepth: 3 +:caption: Topics +:hidden: + +prior-art +``` + +```{toctree} +:maxdepth: 1 +:caption: Workbench +:hidden: + +development +changes +backlog +``` + + +[CrateDB]: https://cratedb.com/docs/guide/home/ +[Kotori]: https://kotori.readthedocs.io/ +[LorryStream]: https://lorrystream.readthedocs.io/ + +[badge-coverage]: https://codecov.io/gh/daq-tools/commons-codec/branch/main/graph/badge.svg +[badge-documentation]: https://img.shields.io/readthedocs/commons-codec +[badge-downloads-per-month]: https://pepy.tech/badge/commons-codec/month +[badge-license]: https://img.shields.io/github/license/daq-tools/commons-codec.svg +[badge-package-version]: https://img.shields.io/pypi/v/commons-codec.svg +[badge-python-versions]: https://img.shields.io/pypi/pyversions/commons-codec.svg +[badge-status]: https://img.shields.io/pypi/status/commons-codec.svg +[badge-tests]: https://github.com/daq-tools/commons-codec/actions/workflows/tests.yml/badge.svg +[project-changelog]: https://commons-codec.readthedocs.io/changes.html +[project-codecov]: https://codecov.io/gh/daq-tools/commons-codec +[project-documentation]: https://commons-codec.readthedocs.io/ +[project-downloads]: https://pepy.tech/project/commons-codec/ +[project-issues]: https://github.com/daq-tools/commons-codec/issues +[project-license]: https://github.com/daq-tools/commons-codec/blob/main/LICENSE +[project-pypi]: https://pypi.org/project/commons-codec +[project-repository]: https://github.com/daq-tools/commons-codec +[project-tests]: https://github.com/daq-tools/commons-codec/actions/workflows/tests.yml diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/doc/prior-art.md b/doc/prior-art.md new file mode 100644 index 0000000..6675161 --- /dev/null +++ b/doc/prior-art.md @@ -0,0 +1,6 @@ +# Prior Art + +- [core-cdc] by Alejandro Cora González + + +[core-cdc]: https://pypi.org/project/core-cdc/ diff --git a/doc/readme.md b/doc/readme.md new file mode 120000 index 0000000..32d46ee --- /dev/null +++ b/doc/readme.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ae613fa..6747177 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,15 @@ optional-dependencies.develop = [ "ruff<0.6", "validate-pyproject<0.19", ] +optional-dependencies.doc = [ + "furo==2024.1.29", # Check visited link style on newer versions! + "myst-parser[linkify]>=0.18,<4", + "sphinx-autobuild==2021.3.14", # Newer versions stopped "watching" appropriately? + "sphinx-copybutton", + "sphinx-design-elements<1", + "sphinxcontrib-mermaid<1", + "sphinxext-opengraph<1", +] optional-dependencies.mongodb = [ "pymongo<4.9", ] @@ -125,9 +134,9 @@ optional-dependencies.test = [ "pytest-mock<4", ] -urls.Changelog = "https://github.com/daq-tools/commons-codec/blob/main/CHANGES.md" -urls.Documentation = "https://github.com/daq-tools/commons-codec/tree/main/docs" -urls.Homepage = "https://github.com/daq-tools/commons-codec" +urls.Changelog = "https://commons-codec.readthedocs.io/changes.html" +urls.Documentation = "https://commons-codec.readthedocs.io/" +urls.Homepage = "https://commons-codec.readthedocs.io/" urls.Issues = "https://github.com/daq-tools/commons-codec/issues" urls.Repository = "https://github.com/daq-tools/commons-codec" @@ -266,6 +275,16 @@ check = [ "test", ] +doc-autobuild = [ + { cmd = "sphinx-autobuild --open-browser --watch src doc doc/_build" }, +] +doc-html = [ + { cmd = "sphinx-build -W --keep-going doc doc/_build" }, +] +doc-linkcheck = [ + { cmd = "sphinx-build -W --keep-going -b linkcheck doc doc/_build" }, +] + format = [ { cmd = "ruff format ." }, # Configure Ruff not to auto-fix (remove!):