diff --git a/.gitignore b/.gitignore index 219774b01..15d1ae7a8 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,8 @@ src/cozy-realtime # docs/ is generated, no need to have it in the repo docs/ + +*.pyc +*.egg-info +.DS_Store +*.egg diff --git a/generate_config.py b/generate_config.py index 6538ec0b0..ab9b805a9 100644 --- a/generate_config.py +++ b/generate_config.py @@ -3,6 +3,7 @@ - Generate mkdocs.yml from mkdocs.yml.tpl """ +import argparse import yaml import json import os @@ -12,6 +13,8 @@ from collections import OrderedDict, namedtuple import fnmatch import sh +from markdown_sphinxjs import gather_doclets_from_dir + def simple_glob(directory, glob_pattern): matches = [] @@ -73,6 +76,7 @@ def find_entry(tree, name): except IndexError: return None + def walk_dict(d, path=None): if path is None: path = [] @@ -114,16 +118,26 @@ def fetch_external_doc(repository, destination): sh.git('clone', repository, '--depth', '1', '.') +TMP_DIR = '/tmp/cozy_docs' +def ensure_root_tmp_dir(): + sh.mkdir('-p', TMP_DIR) + + +def get_doc_tmp_dir(doc_name): + return osp.join(TMP_DIR, doc_name) + + def fetch_all_external_docs_from_file(filename): + ensure_root_tmp_dir() with open(filename) as f: external_docs = [parse_external_doc_line(l) for l in f] - for name, repository, doc_directory in external_docs: - tmpdir = osp.join('/tmp', name) - print('Fetching %s...' % name) + for docname, repository, doc_directory in external_docs: + print('Fetching %s...' % docname) + tmpdir = get_doc_tmp_dir(docname) fetch_external_doc(repository, tmpdir) - src_dir = osp.join('src', name) + src_dir = osp.join('src', docname) sh.rm('-f', src_dir) - print('Linking %s...' % name) + print('Linking %s...' % docname) sh.ln('-s', osp.join(tmpdir, doc_directory), src_dir) @@ -184,12 +198,24 @@ def replace_toc_placeholders(nav, named_tocs): cur[single_key] = flatten_entry_if_single(toc) -def main(argv): +def ensure_jsdoc_cache_exists(src_dir, force=False): + cache_file = '/tmp/jsdoc_data_cache.json' + gather_doclets_from_dir(src_dir, jsdoc_cache=cache_file, force=force) + + +def main(): OUTSIDE_DOCS = 'OUTSIDE_DOCS' - if '--fetch' in argv: + parser = argparse.ArgumentParser() + parser.add_argument('--fetch', action='store_true', help='Fetch all external repositories') + parser.add_argument('--force-jsdoc', action='store_true', help='Force re-analysis of jsdocs') + args = parser.parse_args() + + if args.fetch: fetch_all_external_docs_from_file(OUTSIDE_DOCS) + ensure_jsdoc_cache_exists(TMP_DIR, force=args.force_jsdoc) + with open('./mkdocs.yml.tpl') as f: data = ordered_load(f, yaml.SafeLoader) @@ -219,4 +245,4 @@ def main(argv): warning = '#\n# THIS FILE IS AUTOMATICALLY GENERATED, PLEASE EDIT `mkdocs.yml.tpl` AND LAUNCH `python generate_config.py`\n#\n' f.write(warning + content) -main(sys.argv) +main() diff --git a/markdown_sphinxjs/README.md b/markdown_sphinxjs/README.md new file mode 100644 index 000000000..27658b55e --- /dev/null +++ b/markdown_sphinxjs/README.md @@ -0,0 +1,8 @@ +Markdown SphinxJS +================= + +Auto import JSDoc in your Markdown documentation. + +This is based on the wonderful [SphinxJS] that does the same for the Sphinx documentation generator. + +[SphinxJS]: https://github.com/mozilla/sphinx-js/ diff --git a/markdown_sphinxjs/markdown_sphinxjs/__init__.py b/markdown_sphinxjs/markdown_sphinxjs/__init__.py new file mode 100644 index 000000000..62bb01dfb --- /dev/null +++ b/markdown_sphinxjs/markdown_sphinxjs/__init__.py @@ -0,0 +1,185 @@ +from __future__ import absolute_import +from __future__ import unicode_literals +import re +import os +import json +import tempfile + +from markdown.extensions import Extension +from markdown.blockprocessors import BlockProcessor +from markdown.util import etree + +from sphinx_js import gather_doclets + + +class Config: + def __init__(self, **attrs): + for key, val in attrs.items(): + setattr(self, key, val) + + +class App: + """Make alike for Sphinx app for SphinxJS to work""" + + def __init__(self, config): + self.config = Config(**config) + self._sphinxjs_doclets_by_class = {} + self.confdir = "/tmp" + + +DEFAULT_JSDOC_CONFIG = { + "opts": { + "recurse": True + }, + "source": { + "includePattern": ".+\\.js(doc)?x?$", + "excludePattern": "((^|\\/|\\\\)_)|(min)|(dist)", + "exclude": [ + "node_modules", + "plugins" + ] + } +} + + +def gather_doclets_from_dir(src_dir, jsdoc_cache=None, force=False): + if force and os.path.isfile(jsdoc_cache): + os.unlink(jsdoc_cache) + + with tempfile.NamedTemporaryFile(mode='w', delete=False) as configfile: + configfile.write(json.dumps(DEFAULT_JSDOC_CONFIG, indent=2)) + configfile.seek(0) + app = App( + { + "js_source_path": src_dir, + "js_language": "javascript", + "root_for_relative_js_paths": src_dir, + "jsdoc_config_path": configfile.name, + "jsdoc_cache": jsdoc_cache, + "sphinx_js_lax": True + } + ) + gather_doclets(app) + return { + "by_class": app._sphinxjs_doclets_by_class, + "by_path": app._sphinxjs_doclets_by_path, + } + + +def make_definition_node(ancestor, definition, path): + div = etree.SubElement(ancestor, "div") + div.attrib["class"] = "markdown-sphinxjs-description" + + name = etree.SubElement(div, "h4") + name.text = "%s.%s(%s) => %s" % ( + definition["memberof"], + definition["name"], + ", ".join(definition["meta"]["code"]["paramnames"]), + definition["returns"][0]["type"]["names"][0] + ) + p = etree.SubElement(div, "p") + p.text = definition["description"] + param_table = etree.SubElement(div, "table") + param_head = etree.SubElement(param_table, "thead") + head_row = etree.SubElement(param_table, "tr") + name = etree.SubElement(head_row, "th") + name.text = 'Parameter' + type = etree.SubElement(head_row, "th") + type.text = 'Type' + desc = etree.SubElement(head_row, "th") + desc.text = 'Description' + + # data = etree.SubElement(div, "pre") + # data.text = json.dumps(definition, indent=2) + + params = etree.SubElement(param_table, "tbody") + for param in definition["params"]: + row = etree.SubElement(params, "tr") + name = etree.SubElement(row, "td") + name.text = param["name"] + type = etree.SubElement(row, "td") + type.text = ", ".join(param["type"]["names"]) + desc = etree.SubElement(row, "td") + desc.text = param["description"] + + for example in definition["examples"]: + example_node = etree.SubElement(div, "pre") + example_node.text = """%s""" % example + + return div + + +class MarkdownJSExtension(Extension): + doclets = None + def __init__(self, directory, jsdoc_cache, **kwargs): + super(MarkdownJSExtension, self).__init__(**kwargs) + self.config = {"directory": directory} + self.index = {} + if not MarkdownJSExtension.doclets: + # Markdown extensions are instantiated for each file processed but doclets + # gathering is costly and we do not want to do it every time, this is why + # we store the generated doclets in the class. + # This is a hacky way to have doclets computation be done only once. + MarkdownJSExtension.doclets = gather_doclets_from_dir(directory, jsdoc_cache) + self.doclets = MarkdownJSExtension.doclets + + def extendMarkdown(self, md, **kwargs): + md.registerExtension(self) + + md.parser.blockprocessors.register( + MarkdownJSProcessor(md.parser, self.doclets), "markdown-sphinxjs", 105 + ) + + +class MarkdownJSProcessor(BlockProcessor): + """ + Understands blocks beginining by ---> + The arrow must be followed by an identifier for a function. + + Finds the function referenced by identifier and outputs a div with the description + and parameters of the function. + + Mostly copied from admonition block processor""" + + RE = re.compile(r'^---> ?([\w\-/#]+(?: +[\w\-#]+)*)(?:\n|$)') + RE_SPACES = re.compile(" +") + + def __init__(self, parser, doclets): + super(MarkdownJSProcessor, self).__init__(parser) + self.doclets = doclets + + def test(self, parent, block): + sibling = self.lastChild(parent) + return self.RE.search(block) + + def build(self, ancestor, match): + path_tokens = match.group(1).split(' ') + definition, path = self.doclets["by_path"].get_with_path(path_tokens) + return make_definition_node(ancestor, definition, path) + + def run(self, parent, blocks): + sibling = self.lastChild(parent) + block = blocks.pop(0) + m = self.RE.search(block) + + if m: + block = block[m.end() :] # removes the first line + + block, theRest = self.detab(block) + + if m: + div = self.build(parent, m) + else: + div = sibling + + self.parser.parseChunk(div, block) + + if theRest: + # This block contained unindented line(s) after the first indented + # line. Insert these lines as the first block of the master blocks + # list for future processing. + blocks.insert(0, theRest) + + +def makeExtension(**kwargs): # pragma: no cover + return MarkdownJSExtension(**kwargs) diff --git a/markdown_sphinxjs/requirements.txt b/markdown_sphinxjs/requirements.txt new file mode 100644 index 000000000..56e80423b --- /dev/null +++ b/markdown_sphinxjs/requirements.txt @@ -0,0 +1 @@ +markdown==3.1.1 diff --git a/markdown_sphinxjs/setup.py b/markdown_sphinxjs/setup.py new file mode 100755 index 000000000..a6070df0a --- /dev/null +++ b/markdown_sphinxjs/setup.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Note: To use the 'upload' functionality of this file, you must: +# $ pip install twine + +import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = 'markdownsphinxjs' +DESCRIPTION = 'SphinxJS for markdown' +URL = 'https://github.com/cozy/markdown_sphinxjs' +EMAIL = 'contact@cozycloud.cc' +AUTHOR = 'ptbrowne' +REQUIRES_PYTHON = '>=3.4.3' +VERSION = '1.0.0' + +# What packages are required for this module to be executed? +REQUIRED = ['markdown', 'sphinx_js'] + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + with open(os.path.join(here, NAME, '__version__.py')) as f: + exec(f.read(), about) +else: + about['__version__'] = VERSION + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = 'Build and publish the package.' + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print('\033[1m{0}\033[0m'.format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status('Removing previous builds…') + rmtree(os.path.join(here, 'dist')) + except OSError: + pass + + self.status('Building Source and Wheel (universal) distribution…') + os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) + + self.status('Uploading the package to PyPi via Twine…') + os.system('twine upload dist/*') + + self.status('Pushing git tags…') + os.system('git tag v{0}'.format(about['__version__'])) + os.system('git push --tags') + + sys.exit() + +print(find_packages(exclude=('tests',))) +# Where the magic happens: +setup( + name=NAME, + version=about['__version__'], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type='text/markdown', + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(exclude=('tests',)), + # If your package is a single module, use this instead of 'packages': + #py_modules=['markdownsphinxjs'], + #entry_points={'console_scripts': ['mint=mint.cli:main']}, + install_requires=REQUIRED, + include_package_data=True, + license='MIT', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + ], + # $ setup.py publish support. + cmdclass={'upload': UploadCommand}, +) diff --git a/markdown_sphinxjs/test.py b/markdown_sphinxjs/test.py new file mode 100644 index 000000000..0800cc301 --- /dev/null +++ b/markdown_sphinxjs/test.py @@ -0,0 +1,20 @@ +import markdown +from jsdoc_reference import JSDocReferenceExtension + +ext = JSDocReferenceExtension(directory='/Users/cozy/code/cozy/konnector-libs/packages/cozy-konnector-libs/src') +m = markdown.Markdown(extensions=[ext]) + +html = m.convert( + """ +## Hello + +How are you ? + +--->findDuplicates + + +!!!!note + salut +""" +) +print(html) diff --git a/mkdocs.yml.tpl b/mkdocs.yml.tpl index c50d66cad..fe8733d7e 100644 --- a/mkdocs.yml.tpl +++ b/mkdocs.yml.tpl @@ -78,6 +78,9 @@ markdown_extensions: - smarty - toc: permalink: true +- markdown_sphinxjs: + directory: /tmp/cozy_docs + jsdoc_cache: /tmp/jsdoc_data_cache.json extra: search: tokenizer: "[^a-z\u0430-\u044F\u04510-9\\-\\.]" diff --git a/requirements.txt b/requirements.txt index 5798d74b2..39d2e5898 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,4 @@ Unidecode==1.0.22 urllib3==1.23 mkdocs==1.0.4 mkdocs-material==4.4.0 +git+git://github.com/ptbrowne/sphinx-js@master#egg=sphinxjs diff --git a/src/css/extra.css b/src/css/extra.css index 9960959b8..8147c0c1f 100644 --- a/src/css/extra.css +++ b/src/css/extra.css @@ -39,4 +39,12 @@ /* stack pattern */ .home-action a > * + * { margin-bottom: 1rem -} +} + +/* From markdown-sphinxjs */ +.markdown-sphinxjs-description { + border-top-left-radius: 2px; + border-bottom-left-radius: 2px; + border-left: 0.25rem #ccc solid; + padding-left: 1rem; +} diff --git a/src/tutorials/konnector/getting-started.md b/src/tutorials/konnector/getting-started.md index 27fcf72c3..63a7164b5 100644 --- a/src/tutorials/konnector/getting-started.md +++ b/src/tutorials/konnector/getting-started.md @@ -7,6 +7,8 @@ The easiest way to create a new connector is to use [cozy-konnector-template](https://github.com/konnectors/cozy-konnector-template). +---> BaseKonnector# waitForTwoFaCode + First of all, [download](https://github.com/konnectors/cozy-konnector-template/archive/master.zip) or clone the repository: ```sh