Merge pull request #126 from elchupanebrej/feature/js_markdown_parser

Add Markdown features definitions support using js gherkin implementation

.github/workflows/main.yml

@@ -44,7 +44,11 @@ jobs:
           pip install -U setuptools
           pip install "tox>=4.0" "tox-gh-actions>=3.2" codecov
       - name: Install npm dependencies
-        run: npm install --global
+        run: |
+          npm install "@cucumber/gherkin"
+          npm install "@cucumber/html-formatter"
+          npm install cucumber-html-reporter
+          npm list
       - name: Test with tox
         if: >
           !((matrix.python-version == '3.12' ||

CHANGES.rst

@@ -10,6 +10,9 @@ In-progress
 
 Planned
 -------
+- Refactor internal parser API: split loader and parser APIs
+- Add mobile readthedocs site support theme
+- Convert e2e test features definitions to markdown
 - API doc
 - Add struct_bdd autoload
 - Move tox.ini, pytest.ini into pyproject.toml
@@ -23,10 +26,9 @@ Planned
 - Nested Rules support
 
   - Review after fix https://github.com/cucumber/gherkin/issues/126
-- Implement support of \*.md files
+- Continue support of \*.md files
 
   - Waiting for upstream issue https://github.com/cucumber/gherkin/pull/64
-  - Use js implementation for such feature
 - Support of messages:
 
   - Pending:
@@ -64,6 +66,7 @@ Planned
 
 Unreleased
 ----------
+- Implement support of `Markdown <https://github.com/cucumber/gherkin/blob/main/MARKDOWN_WITH_GHERKIN.md>`_ using js based parser
 - Update versions:
  - Drop python 3.8
  - Add python 3.13

README.rst

@@ -97,6 +97,13 @@ Install pytest-bdd-ng
 
     pip install pytest-bdd-ng
 
+Install extra packages for parsing Markdown defined features
+############################################################
+
+.. code-block:: console
+
+    npm install @cucumber/gherkin
+
 Install extra packages for reporting
 ####################################
 

features/Feature/Load/Autoload.feature

@@ -11,19 +11,30 @@ Feature: Gherkin features autoload
           Scenario: Passing scenario
             * Passing step
         """
+      Given File "Another.passing.feature.md" with content:
+        """markdown
+        # Feature: Passing feature
+        ## Scenario: Passing scenario
+
+        * Given Passing step
+        """
+
+      Given Install npm packages
+      |packages|@cucumber/gherkin|
+
       And File "conftest.py" with content:
         """python
         from pytest_bdd import step
 
         @step('Passing step')
-        def plain_step():
+        def _():
           ...
         """
     Scenario: Feature is loaded by default
       When run pytest
       Then pytest outcome must contain tests with statuses:
         |passed|
-        |     1|
+        |     2|
 
     Scenario: Feature autoload could be disabled via command line
       When run pytest

setup.cfg

@@ -10,3 +10,5 @@ where = src
 [options.package_data]
 pytest_bdd.template =
     test.py.mak
+pytest_bdd =
+    markdown_parser.js

src/pytest_bdd/markdown_parser.js

@@ -0,0 +1,13 @@
+var Gherkin = require("@cucumber/gherkin");
+var Messages = require("@cucumber/messages");
+var fs = require('fs');
+
+const feature = fs.readFileSync(0, "utf-8");
+
+var uuidFn = Messages.IdGenerator.uuid();
+var builder = new Gherkin.AstBuilder(uuidFn);
+var matcher = new Gherkin.GherkinInMarkdownTokenMatcher();
+
+var parser = new Gherkin.Parser(builder, matcher);
+var gherkinDocument = parser.parse(feature);
+console.log(JSON.stringify(gherkinDocument));

src/pytest_bdd/parser.py

@@ -1,17 +1,22 @@
+import json
 import linecache
 from collections.abc import Sequence
+from contextlib import ExitStack
 from functools import partial
 from itertools import filterfalse
-from operator import contains, itemgetter, methodcaller
+from operator import contains, itemgetter
 from pathlib import Path
-from typing import Callable, List, Set, Tuple, Union
+from shutil import which
+from subprocess import check_output
+from typing import Callable, Union
 
 from attr import attrib, attrs
 from gherkin.ast_builder import AstBuilder
 from gherkin.errors import CompositeParserException
 from gherkin.parser import Parser as CucumberIOBaseParser  # type: ignore[import]
 from gherkin.pickles.compiler import Compiler as PicklesCompiler
 
+from pytest_bdd.compatibility.importlib.resources import as_file, files
 from pytest_bdd.compatibility.parser import ParserProtocol
 from pytest_bdd.compatibility.path import relpath
 from pytest_bdd.compatibility.pytest import Config
@@ -26,13 +31,29 @@
     assert StructBDDParser  # type: ignore[truthy-function]
 
 
-@attrs
-class GlobMixin:
-    glob: Callable[..., Sequence[Union[str, Path]]] = attrib(default=methodcaller("glob", "*.feature"), kw_only=True)
+class BaseParser(ParserProtocol):
+    def build_feature(self, gherkin_document_raw_dict, filename: str) -> Feature:
+        gherkin_document = Feature.load_gherkin_document(gherkin_document_raw_dict)
+
+        pickles_data = PicklesCompiler(id_generator=self.id_generator).compile(gherkin_document_raw_dict)
+        pickles = Feature.load_pickles(pickles_data)
+
+        feature = Feature(  # type: ignore[call-arg]
+            gherkin_document=gherkin_document,
+            uri=gherkin_document.uri,
+            pickles=pickles,
+            filename=filename,
+        )
+
+        return feature
 
 
 @attrs
-class GherkinParser(GlobMixin, ParserProtocol):
+class GherkinParser(BaseParser):
+    glob: Callable[..., Sequence[Union[str, Path]]] = attrib(
+        default=lambda path: path.glob("*.feature") + path.glob("*.gherkin"), kw_only=True
+    )
+
     def parse(
         self, config: Union[Config, PytestBDDIdGeneratorHandler], path: Path, uri: str, *args, **kwargs
     ) -> tuple[Feature, str]:
@@ -59,6 +80,7 @@ def parse(
         )
         return feature, feature_file_data
 
+    # TODO Move out of parser to loader component
     def get_from_paths(self, config: Config, paths: Sequence[Path], **kwargs) -> Sequence[Feature]:
         """Get features for given paths."""
         seen_names: set[Path] = set()
@@ -89,17 +111,26 @@ def get_from_paths(self, config: Config, paths: Sequence[Path], **kwargs) -> Seq
 
         return sorted(features, key=lambda feature: feature.name or feature.filename)
 
-    def build_feature(self, gherkin_document_raw_dict, filename: str) -> Feature:
-        gherkin_document = Feature.load_gherkin_document(gherkin_document_raw_dict)
 
-        pickles_data = PicklesCompiler(id_generator=self.id_generator).compile(gherkin_document_raw_dict)
-        pickles = Feature.load_pickles(pickles_data)
+@attrs
+class MarkdownGherkinParser(BaseParser):
+    glob: Callable[..., Sequence[Union[str, Path]]] = attrib(
+        default=lambda path: path.glob("*.feature.md") + path.glob("*.gherkin.md"), kw_only=True
+    )
 
-        feature = Feature(  # type: ignore[call-arg]
-            gherkin_document=gherkin_document,
-            uri=gherkin_document.uri,
-            pickles=pickles,
-            filename=filename,
-        )
+    def parse(
+        self, config: Union[Config, PytestBDDIdGeneratorHandler], path: Path, uri: str, *args, **kwargs
+    ) -> tuple[Feature, str]:
+        with ExitStack() as stack:
+            feature_file, script_path = [
+                stack.enter_context(path.open(mode="rb")),
+                stack.enter_context(as_file(files("pytest_bdd").joinpath("markdown_parser.js"))),
+            ]
+            gherkin_document_raw_dict = json.loads(check_output([which("node") or "", script_path], stdin=feature_file))
+        gherkin_document_raw_dict["uri"] = uri
 
-        return feature
+        feature = self.build_feature(
+            gherkin_document_raw_dict,
+            filename=str(path.as_posix()),
+        )
+        return feature, path.read_text()

src/pytest_bdd/plugin.py

@@ -34,7 +34,8 @@
 from pytest_bdd.message_plugin import MessagePlugin
 from pytest_bdd.mimetypes import Mimetype
 from pytest_bdd.model import Feature
-from pytest_bdd.parser import GherkinParser
+from pytest_bdd.npm_resource import check_npm, check_npm_package
+from pytest_bdd.parser import GherkinParser, MarkdownGherkinParser
 from pytest_bdd.parsers import cucumber_expression
 from pytest_bdd.reporting import ScenarioReporterPlugin
 from pytest_bdd.runner import ScenarioRunner
@@ -344,14 +345,31 @@ def pytest_bdd_match_step_definition_to_step(request, feature, scenario, step, p
 
 
 def pytest_bdd_get_mimetype(config: Config, path: Path):
+    # TODO use mimetypes module
     if str(path).endswith(".gherkin") or str(path).endswith(".feature"):
         return Mimetype.gherkin_plain.value
+    elif str(path).endswith(".gherkin.md") or str(path).endswith(".feature.md"):
+        if not check_npm():
+            return
+
+        if not any(
+            [
+                check_npm_package("@cucumber/gherkin", global_install=True),
+                check_npm_package("@cucumber/gherkin"),
+            ]
+        ):
+            return
+        return Mimetype.markdown.value
 
 
 def pytest_bdd_get_parser(config: Config, mimetype: str):
-    return {Mimetype.gherkin_plain.value: GherkinParser}.get(mimetype)
+    return {
+        Mimetype.gherkin_plain.value: GherkinParser,
+        Mimetype.markdown.value: MarkdownGherkinParser,
+    }.get(mimetype)
 
 
 def pytest_bdd_is_collectible(config: Config, path: Path):
+    # TODO add more extensions
     if any(map(partial(contains, {".gherkin", ".feature", ".url", ".desktop", ".webloc"}), path.suffixes)):
         return True

tests/e2e/conftest.py

@@ -26,7 +26,7 @@ def httpserver_port(httpserver):
     return httpserver.port
 
 
-@given(re.compile('File "(?P<name>\\w+)(?P<extension>\\.\\w+)" with (?P<extra_opts>.*|\\s)content:'))
+@given(re.compile(r"File \"(?P<name>(\.|\w)+)(?P<extension>\.\w+)\" with (?P<extra_opts>.*|\s)content:"))
 def write_file_with_extras(name, extension, testdir, step, request, extra_opts, tmp_path):
     content = step.doc_string.content
     is_fixture_templated = "fixture templated" in extra_opts
@@ -36,8 +36,8 @@ def write_file_with_extras(name, extension, testdir, step, request, extra_opts,
         format_options = dict(
             map(lambda fixture_name: (fixture_name, str(request.getfixturevalue(fixture_name))), template_fields)
         )
-    makefile_arg = {name: str(content).format_map(format_options) if is_fixture_templated else content}
-    testdir.makefile(extension, **makefile_arg)
+    file_data = str(content).format_map(format_options) if is_fixture_templated else content
+    (Path(testdir.tmpdir.strpath) / f"{name}{extension}").write_text(file_data, encoding="utf-8")
 
 
 @given(