WIP Reference docstrings for markdown documentations

.gitignore

@@ -26,3 +26,8 @@ src/cozy-realtime
 
 # docs/ is generated, no need to have it in the repo
 docs/
+
+*.pyc
+*.egg-info
+.DS_Store
+*.egg

generate_config.py

@@ -3,6 +3,7 @@
 - Generate mkdocs.yml from mkdocs.yml.tpl
 """
 
+import argparse
 import yaml
 import json
 import os
@@ -12,6 +13,8 @@
 from collections import OrderedDict, namedtuple
 import fnmatch
 import sh
+from markdown_sphinxjs import gather_doclets_from_dir
+
 
 def simple_glob(directory, glob_pattern):
     matches = []
@@ -73,6 +76,7 @@ def find_entry(tree, name):
     except IndexError:
         return None
 
+
 def walk_dict(d, path=None):
     if path is None:
         path = []
@@ -114,16 +118,26 @@ def fetch_external_doc(repository, destination):
             sh.git('clone', repository, '--depth', '1', '.')
 
 
+TMP_DIR = '/tmp/cozy_docs'
+def ensure_root_tmp_dir():
+    sh.mkdir('-p', TMP_DIR)
+
+
+def get_doc_tmp_dir(doc_name):
+    return osp.join(TMP_DIR, doc_name)
+
+
 def fetch_all_external_docs_from_file(filename):
+    ensure_root_tmp_dir()
     with open(filename) as f:
         external_docs = [parse_external_doc_line(l) for l in f]
-    for name, repository, doc_directory in external_docs:
-        tmpdir = osp.join('/tmp', name)
-        print('Fetching %s...' % name)
+    for docname, repository, doc_directory in external_docs:
+        print('Fetching %s...' % docname)
+        tmpdir = get_doc_tmp_dir(docname)
         fetch_external_doc(repository, tmpdir)
-        src_dir = osp.join('src', name)
+        src_dir = osp.join('src', docname)
         sh.rm('-f', src_dir)
-        print('Linking %s...' % name)
+        print('Linking %s...' % docname)
         sh.ln('-s', osp.join(tmpdir, doc_directory), src_dir)
 
 
@@ -184,12 +198,24 @@ def replace_toc_placeholders(nav, named_tocs):
             cur[single_key] = flatten_entry_if_single(toc)
 
 
-def main(argv):
+def ensure_jsdoc_cache_exists(src_dir, force=False):
+    cache_file = '/tmp/jsdoc_data_cache.json'
+    gather_doclets_from_dir(src_dir, jsdoc_cache=cache_file, force=force)
+
+
+def main():
     OUTSIDE_DOCS = 'OUTSIDE_DOCS'
 
-    if '--fetch' in argv:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--fetch', action='store_true', help='Fetch all external repositories')
+    parser.add_argument('--force-jsdoc', action='store_true', help='Force re-analysis of jsdocs')
+    args = parser.parse_args()
+
+    if args.fetch:
         fetch_all_external_docs_from_file(OUTSIDE_DOCS)
 
+    ensure_jsdoc_cache_exists(TMP_DIR, force=args.force_jsdoc)
+
     with open('./mkdocs.yml.tpl') as f:
         data = ordered_load(f, yaml.SafeLoader)
 
@@ -219,4 +245,4 @@ def main(argv):
         warning = '#\n# THIS FILE IS AUTOMATICALLY GENERATED, PLEASE EDIT `mkdocs.yml.tpl` AND LAUNCH `python generate_config.py`\n#\n'
         f.write(warning + content)
 
-main(sys.argv)
+main()

markdown_sphinxjs/README.md

@@ -0,0 +1,8 @@
+Markdown SphinxJS
+=================
+
+Auto import JSDoc in your Markdown documentation.
+
+This is based on the wonderful [SphinxJS] that does the same for the Sphinx documentation generator.
+
+[SphinxJS]: https://github.com/mozilla/sphinx-js/

markdown_sphinxjs/markdown_sphinxjs/__init__.py

@@ -0,0 +1,185 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+import re
+import os
+import json
+import tempfile
+
+from markdown.extensions import Extension
+from markdown.blockprocessors import BlockProcessor
+from markdown.util import etree
+
+from sphinx_js import gather_doclets
+
+
+class Config:
+    def __init__(self, **attrs):
+        for key, val in attrs.items():
+            setattr(self, key, val)
+
+
+class App:
+    """Make alike for Sphinx app for SphinxJS to work"""
+
+    def __init__(self, config):
+        self.config = Config(**config)
+        self._sphinxjs_doclets_by_class = {}
+        self.confdir = "/tmp"
+
+
+DEFAULT_JSDOC_CONFIG = {
+  "opts": {
+    "recurse": True
+  },
+  "source": {
+    "includePattern": ".+\\.js(doc)?x?$",
+    "excludePattern": "((^|\\/|\\\\)_)|(min)|(dist)",
+    "exclude": [
+      "node_modules",
+      "plugins"
+    ]
+  }
+}
+
+
+def gather_doclets_from_dir(src_dir, jsdoc_cache=None, force=False):
+    if force and os.path.isfile(jsdoc_cache):
+        os.unlink(jsdoc_cache)
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as configfile:
+        configfile.write(json.dumps(DEFAULT_JSDOC_CONFIG, indent=2))
+        configfile.seek(0)
+        app = App(
+            {
+                "js_source_path": src_dir,
+                "js_language": "javascript",
+                "root_for_relative_js_paths": src_dir,
+                "jsdoc_config_path": configfile.name,
+                "jsdoc_cache": jsdoc_cache,
+                "sphinx_js_lax": True
+            }
+        )
+        gather_doclets(app)
+    return {
+        "by_class": app._sphinxjs_doclets_by_class,
+        "by_path": app._sphinxjs_doclets_by_path,
+    }
+
+
+def make_definition_node(ancestor, definition, path):
+    div = etree.SubElement(ancestor, "div")
+    div.attrib["class"] = "markdown-sphinxjs-description"
+
+    name = etree.SubElement(div, "h4")
+    name.text = "%s.%s(%s) => %s" % (
+        definition["memberof"],
+        definition["name"],
+        ", ".join(definition["meta"]["code"]["paramnames"]),
+        definition["returns"][0]["type"]["names"][0]
+    )
+    p = etree.SubElement(div, "p")
+    p.text = definition["description"]
+    param_table = etree.SubElement(div, "table")
+    param_head = etree.SubElement(param_table, "thead")
+    head_row = etree.SubElement(param_table, "tr")
+    name = etree.SubElement(head_row, "th")
+    name.text = 'Parameter'
+    type = etree.SubElement(head_row, "th")
+    type.text = 'Type'
+    desc = etree.SubElement(head_row, "th")
+    desc.text = 'Description'
+
+    # data = etree.SubElement(div, "pre")
+    # data.text = json.dumps(definition, indent=2)
+
+    params = etree.SubElement(param_table, "tbody")
+    for param in definition["params"]:
+        row = etree.SubElement(params, "tr")
+        name = etree.SubElement(row, "td")
+        name.text = param["name"]
+        type = etree.SubElement(row, "td")
+        type.text = ", ".join(param["type"]["names"])
+        desc = etree.SubElement(row, "td")
+        desc.text = param["description"]
+
+    for example in definition["examples"]:
+        example_node = etree.SubElement(div, "pre")
+        example_node.text = """%s""" % example
+
+    return div
+
+
+class MarkdownJSExtension(Extension):
+    doclets = None
+    def __init__(self, directory, jsdoc_cache, **kwargs):
+        super(MarkdownJSExtension, self).__init__(**kwargs)
+        self.config = {"directory": directory}
+        self.index = {}
+        if not MarkdownJSExtension.doclets:
+            # Markdown extensions are instantiated for each file processed but doclets
+            # gathering is costly and we do not want to do it every time, this is why
+            # we store the generated doclets in the class. 
+            # This is a hacky way to have doclets computation be done only once.
+            MarkdownJSExtension.doclets = gather_doclets_from_dir(directory, jsdoc_cache)
+        self.doclets = MarkdownJSExtension.doclets
+
+    def extendMarkdown(self, md, **kwargs):
+        md.registerExtension(self)
+
+        md.parser.blockprocessors.register(
+            MarkdownJSProcessor(md.parser, self.doclets), "markdown-sphinxjs", 105
+        )
+
+
+class MarkdownJSProcessor(BlockProcessor):
+    """
+    Understands blocks beginining by --->
+    The arrow must be followed by an identifier for a function.
+
+    Finds the function referenced by identifier and outputs a div with the description
+    and parameters of the function.
+
+    Mostly copied from admonition block processor"""
+
+    RE = re.compile(r'^---> ?([\w\-/#]+(?: +[\w\-#]+)*)(?:\n|$)')
+    RE_SPACES = re.compile("  +")
+
+    def __init__(self, parser, doclets):
+        super(MarkdownJSProcessor, self).__init__(parser)
+        self.doclets = doclets
+
+    def test(self, parent, block):
+        sibling = self.lastChild(parent)
+        return self.RE.search(block)
+
+    def build(self, ancestor, match):
+        path_tokens = match.group(1).split(' ')
+        definition, path = self.doclets["by_path"].get_with_path(path_tokens)
+        return make_definition_node(ancestor, definition, path)
+
+    def run(self, parent, blocks):
+        sibling = self.lastChild(parent)
+        block = blocks.pop(0)
+        m = self.RE.search(block)
+
+        if m:
+            block = block[m.end() :]  # removes the first line
+
+        block, theRest = self.detab(block)
+
+        if m:
+            div = self.build(parent, m)
+        else:
+            div = sibling
+
+        self.parser.parseChunk(div, block)
+
+        if theRest:
+            # This block contained unindented line(s) after the first indented
+            # line. Insert these lines as the first block of the master blocks
+            # list for future processing.
+            blocks.insert(0, theRest)
+
+
+def makeExtension(**kwargs):  # pragma: no cover
+    return MarkdownJSExtension(**kwargs)

markdown_sphinxjs/requirements.txt

@@ -0,0 +1 @@
+markdown==3.1.1