From 8cdac94e2e522b77561237b095abe9e43ba03356 Mon Sep 17 00:00:00 2001 From: Kostiantyn Goloveshko Date: Sun, 3 Nov 2024 14:41:29 +0200 Subject: [PATCH] Add Markdown features definitions support using js gherkin implementation --- .github/workflows/main.yml | 6 +- CHANGES.rst | 7 +- README.rst | 7 + docs/features.rst | 346 ++++++++++++------------- features/Feature/Load/Autoload.feature | 15 +- package.json | 7 - setup.cfg | 2 + src/pytest_bdd/markdown_parser.js | 13 + src/pytest_bdd/parser.py | 65 +++-- src/pytest_bdd/plugin.py | 22 +- tests/e2e/conftest.py | 6 +- 11 files changed, 289 insertions(+), 207 deletions(-) delete mode 100644 package.json create mode 100644 src/pytest_bdd/markdown_parser.js diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e36863a6..dcc9411e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,11 @@ jobs: pip install -U setuptools pip install "tox>=4.0" "tox-gh-actions>=3.2" codecov - name: Install npm dependencies - run: npm install --global + run: | + npm install "@cucumber/gherkin" + npm install "@cucumber/html-formatter" + npm install cucumber-html-reporter + npm list - name: Test with tox if: > !((matrix.python-version == '3.12' || diff --git a/CHANGES.rst b/CHANGES.rst index 0eb47873..84980018 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,9 @@ In-progress Planned ------- +- Refactor internal parser API: split loader and parser APIs +- Add mobile readthedocs site support theme +- Convert e2e test features definitions to markdown - API doc - Add struct_bdd autoload - Move tox.ini, pytest.ini into pyproject.toml @@ -23,10 +26,9 @@ Planned - Nested Rules support - Review after fix https://github.com/cucumber/gherkin/issues/126 -- Implement support of \*.md files +- Continue support of \*.md files - Waiting for upstream issue https://github.com/cucumber/gherkin/pull/64 - - Use js implementation for such feature - Support of messages: - Pending: @@ -64,6 +66,7 @@ Planned Unreleased ---------- +- Implement support of `Markdown `_ using js based parser - Update versions: - Drop python 3.8 - Add python 3.13 diff --git a/README.rst b/README.rst index d0eee416..b9da77fa 100644 --- a/README.rst +++ b/README.rst @@ -97,6 +97,13 @@ Install pytest-bdd-ng pip install pytest-bdd-ng +Install extra packages for parsing Markdown defined features +############################################################ + +.. code-block:: console + + npm install @cucumber/gherkin + Install extra packages for reporting #################################### diff --git a/docs/features.rst b/docs/features.rst index 7fe413c5..ea947e19 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -1,173 +1,173 @@ -Features -======== - -.. NOTE:: Features below are part of end-to-end test suite; You always could find most specific - use cases of **pytest-bdd-ng** by investigation of its regression - test suite https://github.com/elchupanebrej/pytest-bdd-ng/tree/default/tests - - - -Feature -------- - -Tag conversion -############## - -.. include:: ../features/Feature/Tag conversion.feature - :code: gherkin - -Tag -### - -.. include:: ../features/Feature/Tag.feature - :code: gherkin - -Description -########### - -.. include:: ../features/Feature/Description.feature - :code: gherkin - -Localization -############ - -.. include:: ../features/Feature/Localization.feature - :code: gherkin - -Load -#### - -Autoload -!!!!!!!! - -.. include:: ../features/Feature/Load/Autoload.feature - :code: gherkin - -Scenario function loader -!!!!!!!!!!!!!!!!!!!!!!!! - -.. include:: ../features/Feature/Load/Scenario function loader.feature - :code: gherkin - -Scenario search from base directory -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -.. include:: ../features/Feature/Load/Scenario search from base directory.feature - :code: gherkin - -Scenario search from base url -!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -.. include:: ../features/Feature/Load/Scenario search from base url.feature - :code: gherkin - -Scenario --------- - -Tag -### - -.. include:: ../features/Scenario/Tag.feature - :code: gherkin - -Description -########### - -.. include:: ../features/Scenario/Description.feature - :code: gherkin - -Outline -####### - -Examples Tag -!!!!!!!!!!!! - -.. include:: ../features/Scenario/Outline/Examples Tag.feature - :code: gherkin - -Report ------- - -Gathering -######### - -.. include:: ../features/Report/Gathering.feature - :code: gherkin - -Tutorial --------- - -Launch -###### - -.. include:: ../features/Tutorial/Launch.feature - :code: gherkin - -Step ----- - -Data table -########## - -.. include:: ../features/Step/Data table.feature - :code: gherkin - -Step definition bounding -######################## - -.. include:: ../features/Step/Step definition bounding.feature - :code: gherkin - -Doc string -########## - -.. include:: ../features/Step/Doc string.feature - :code: gherkin - -Step definition ---------------- - -Target fixtures specification -############################# - -.. include:: ../features/Step definition/Target fixtures specification.feature - :code: gherkin - -Pytest fixtures substitution -############################ - -.. include:: ../features/Step definition/Pytest fixtures substitution.feature - :code: gherkin - -Parameters -########## - -Injection as fixtures -!!!!!!!!!!!!!!!!!!!!! - -.. include:: ../features/Step definition/Parameters/Injection as fixtures.feature - :code: gherkin - -Parsing -!!!!!!! - -.. include:: ../features/Step definition/Parameters/Parsing.feature - :code: gherkin - -Parsing by custom parser -!!!!!!!!!!!!!!!!!!!!!!!! - -.. include:: ../features/Step definition/Parameters/Parsing by custom parser.feature - :code: gherkin - -Conversion -!!!!!!!!!! - -.. include:: ../features/Step definition/Parameters/Conversion.feature - :code: gherkin - -Defaults -!!!!!!!! - -.. include:: ../features/Step definition/Parameters/Defaults.feature - :code: gherkin +Features +======== + +.. NOTE:: Features below are part of end-to-end test suite; You always could find most specific + use cases of **pytest-bdd-ng** by investigation of its regression + test suite https://github.com/elchupanebrej/pytest-bdd-ng/tree/default/tests + + + +Tutorial +-------- + +Launch +###### + +.. include:: ../features/Tutorial/Launch.feature + :code: gherkin + +Step definition +--------------- + +Pytest fixtures substitution +############################ + +.. include:: ../features/Step definition/Pytest fixtures substitution.feature + :code: gherkin + +Target fixtures specification +############################# + +.. include:: ../features/Step definition/Target fixtures specification.feature + :code: gherkin + +Parameters +########## + +Conversion +!!!!!!!!!! + +.. include:: ../features/Step definition/Parameters/Conversion.feature + :code: gherkin + +Defaults +!!!!!!!! + +.. include:: ../features/Step definition/Parameters/Defaults.feature + :code: gherkin + +Injection as fixtures +!!!!!!!!!!!!!!!!!!!!! + +.. include:: ../features/Step definition/Parameters/Injection as fixtures.feature + :code: gherkin + +Parsing by custom parser +!!!!!!!!!!!!!!!!!!!!!!!! + +.. include:: ../features/Step definition/Parameters/Parsing by custom parser.feature + :code: gherkin + +Parsing +!!!!!!! + +.. include:: ../features/Step definition/Parameters/Parsing.feature + :code: gherkin + +Step +---- + +Data table +########## + +.. include:: ../features/Step/Data table.feature + :code: gherkin + +Doc string +########## + +.. include:: ../features/Step/Doc string.feature + :code: gherkin + +Step definition bounding +######################## + +.. include:: ../features/Step/Step definition bounding.feature + :code: gherkin + +Scenario +-------- + +Description +########### + +.. include:: ../features/Scenario/Description.feature + :code: gherkin + +Tag +### + +.. include:: ../features/Scenario/Tag.feature + :code: gherkin + +Outline +####### + +Examples Tag +!!!!!!!!!!!! + +.. include:: ../features/Scenario/Outline/Examples Tag.feature + :code: gherkin + +Report +------ + +Gathering +######### + +.. include:: ../features/Report/Gathering.feature + :code: gherkin + +Feature +------- + +Description +########### + +.. include:: ../features/Feature/Description.feature + :code: gherkin + +Localization +############ + +.. include:: ../features/Feature/Localization.feature + :code: gherkin + +Tag conversion +############## + +.. include:: ../features/Feature/Tag conversion.feature + :code: gherkin + +Tag +### + +.. include:: ../features/Feature/Tag.feature + :code: gherkin + +Load +#### + +Autoload +!!!!!!!! + +.. include:: ../features/Feature/Load/Autoload.feature + :code: gherkin + +Scenario function loader +!!!!!!!!!!!!!!!!!!!!!!!! + +.. include:: ../features/Feature/Load/Scenario function loader.feature + :code: gherkin + +Scenario search from base directory +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. include:: ../features/Feature/Load/Scenario search from base directory.feature + :code: gherkin + +Scenario search from base url +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. include:: ../features/Feature/Load/Scenario search from base url.feature + :code: gherkin diff --git a/features/Feature/Load/Autoload.feature b/features/Feature/Load/Autoload.feature index 6592a171..a1a4def9 100644 --- a/features/Feature/Load/Autoload.feature +++ b/features/Feature/Load/Autoload.feature @@ -11,19 +11,30 @@ Feature: Gherkin features autoload Scenario: Passing scenario * Passing step """ + Given File "Another.passing.feature.md" with content: + """markdown + # Feature: Passing feature + ## Scenario: Passing scenario + + * Given Passing step + """ + + Given Install npm packages + |packages|@cucumber/gherkin| + And File "conftest.py" with content: """python from pytest_bdd import step @step('Passing step') - def plain_step(): + def _(): ... """ Scenario: Feature is loaded by default When run pytest Then pytest outcome must contain tests with statuses: |passed| - | 1| + | 2| Scenario: Feature autoload could be disabled via command line When run pytest diff --git a/package.json b/package.json deleted file mode 100644 index 4b8c9ab1..00000000 --- a/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "pytest_bdd_ng", - "dependencies": { - "@cucumber/html-formatter": "*", - "cucumber-html-reporter": "*" - } -} diff --git a/setup.cfg b/setup.cfg index 7f3d3bb0..860c6bfb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,3 +10,5 @@ where = src [options.package_data] pytest_bdd.template = test.py.mak +pytest_bdd = + markdown_parser.js diff --git a/src/pytest_bdd/markdown_parser.js b/src/pytest_bdd/markdown_parser.js new file mode 100644 index 00000000..44bd2b6d --- /dev/null +++ b/src/pytest_bdd/markdown_parser.js @@ -0,0 +1,13 @@ +var Gherkin = require("@cucumber/gherkin"); +var Messages = require("@cucumber/messages"); +var fs = require('fs'); + +const feature = fs.readFileSync(0, "utf-8"); + +var uuidFn = Messages.IdGenerator.uuid(); +var builder = new Gherkin.AstBuilder(uuidFn); +var matcher = new Gherkin.GherkinInMarkdownTokenMatcher(); + +var parser = new Gherkin.Parser(builder, matcher); +var gherkinDocument = parser.parse(feature); +console.log(JSON.stringify(gherkinDocument)); diff --git a/src/pytest_bdd/parser.py b/src/pytest_bdd/parser.py index 817f54cb..4fa95ad0 100644 --- a/src/pytest_bdd/parser.py +++ b/src/pytest_bdd/parser.py @@ -1,10 +1,14 @@ +import json import linecache from collections.abc import Sequence +from contextlib import ExitStack from functools import partial from itertools import filterfalse -from operator import contains, itemgetter, methodcaller +from operator import contains, itemgetter from pathlib import Path -from typing import Callable, List, Set, Tuple, Union +from shutil import which +from subprocess import check_output +from typing import Callable, Union from attr import attrib, attrs from gherkin.ast_builder import AstBuilder @@ -12,6 +16,7 @@ from gherkin.parser import Parser as CucumberIOBaseParser # type: ignore[import] from gherkin.pickles.compiler import Compiler as PicklesCompiler +from pytest_bdd.compatibility.importlib.resources import as_file, files from pytest_bdd.compatibility.parser import ParserProtocol from pytest_bdd.compatibility.path import relpath from pytest_bdd.compatibility.pytest import Config @@ -26,13 +31,29 @@ assert StructBDDParser # type: ignore[truthy-function] -@attrs -class GlobMixin: - glob: Callable[..., Sequence[Union[str, Path]]] = attrib(default=methodcaller("glob", "*.feature"), kw_only=True) +class BaseParser(ParserProtocol): + def build_feature(self, gherkin_document_raw_dict, filename: str) -> Feature: + gherkin_document = Feature.load_gherkin_document(gherkin_document_raw_dict) + + pickles_data = PicklesCompiler(id_generator=self.id_generator).compile(gherkin_document_raw_dict) + pickles = Feature.load_pickles(pickles_data) + + feature = Feature( # type: ignore[call-arg] + gherkin_document=gherkin_document, + uri=gherkin_document.uri, + pickles=pickles, + filename=filename, + ) + + return feature @attrs -class GherkinParser(GlobMixin, ParserProtocol): +class GherkinParser(BaseParser): + glob: Callable[..., Sequence[Union[str, Path]]] = attrib( + default=lambda path: path.glob("*.feature") + path.glob("*.gherkin"), kw_only=True + ) + def parse( self, config: Union[Config, PytestBDDIdGeneratorHandler], path: Path, uri: str, *args, **kwargs ) -> tuple[Feature, str]: @@ -59,6 +80,7 @@ def parse( ) return feature, feature_file_data + # TODO Move out of parser to loader component def get_from_paths(self, config: Config, paths: Sequence[Path], **kwargs) -> Sequence[Feature]: """Get features for given paths.""" seen_names: set[Path] = set() @@ -89,17 +111,26 @@ def get_from_paths(self, config: Config, paths: Sequence[Path], **kwargs) -> Seq return sorted(features, key=lambda feature: feature.name or feature.filename) - def build_feature(self, gherkin_document_raw_dict, filename: str) -> Feature: - gherkin_document = Feature.load_gherkin_document(gherkin_document_raw_dict) - pickles_data = PicklesCompiler(id_generator=self.id_generator).compile(gherkin_document_raw_dict) - pickles = Feature.load_pickles(pickles_data) +@attrs +class MarkdownGherkinParser(BaseParser): + glob: Callable[..., Sequence[Union[str, Path]]] = attrib( + default=lambda path: path.glob("*.feature.md") + path.glob("*.gherkin.md"), kw_only=True + ) - feature = Feature( # type: ignore[call-arg] - gherkin_document=gherkin_document, - uri=gherkin_document.uri, - pickles=pickles, - filename=filename, - ) + def parse( + self, config: Union[Config, PytestBDDIdGeneratorHandler], path: Path, uri: str, *args, **kwargs + ) -> tuple[Feature, str]: + with ExitStack() as stack: + feature_file, script_path = [ + stack.enter_context(path.open(mode="rb")), + stack.enter_context(as_file(files("pytest_bdd").joinpath("markdown_parser.js"))), + ] + gherkin_document_raw_dict = json.loads(check_output([which("node") or "", script_path], stdin=feature_file)) + gherkin_document_raw_dict["uri"] = uri - return feature + feature = self.build_feature( + gherkin_document_raw_dict, + filename=str(path.as_posix()), + ) + return feature, path.read_text() diff --git a/src/pytest_bdd/plugin.py b/src/pytest_bdd/plugin.py index aac163de..c4001f54 100644 --- a/src/pytest_bdd/plugin.py +++ b/src/pytest_bdd/plugin.py @@ -34,7 +34,8 @@ from pytest_bdd.message_plugin import MessagePlugin from pytest_bdd.mimetypes import Mimetype from pytest_bdd.model import Feature -from pytest_bdd.parser import GherkinParser +from pytest_bdd.npm_resource import check_npm, check_npm_package +from pytest_bdd.parser import GherkinParser, MarkdownGherkinParser from pytest_bdd.parsers import cucumber_expression from pytest_bdd.reporting import ScenarioReporterPlugin from pytest_bdd.runner import ScenarioRunner @@ -344,14 +345,31 @@ def pytest_bdd_match_step_definition_to_step(request, feature, scenario, step, p def pytest_bdd_get_mimetype(config: Config, path: Path): + # TODO use mimetypes module if str(path).endswith(".gherkin") or str(path).endswith(".feature"): return Mimetype.gherkin_plain.value + elif str(path).endswith(".gherkin.md") or str(path).endswith(".feature.md"): + if not check_npm(): + return + + if not any( + [ + check_npm_package("@cucumber/gherkin", global_install=True), + check_npm_package("@cucumber/gherkin"), + ] + ): + return + return Mimetype.markdown.value def pytest_bdd_get_parser(config: Config, mimetype: str): - return {Mimetype.gherkin_plain.value: GherkinParser}.get(mimetype) + return { + Mimetype.gherkin_plain.value: GherkinParser, + Mimetype.markdown.value: MarkdownGherkinParser, + }.get(mimetype) def pytest_bdd_is_collectible(config: Config, path: Path): + # TODO add more extensions if any(map(partial(contains, {".gherkin", ".feature", ".url", ".desktop", ".webloc"}), path.suffixes)): return True diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ace805f6..b7a3b518 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -26,7 +26,7 @@ def httpserver_port(httpserver): return httpserver.port -@given(re.compile('File "(?P\\w+)(?P\\.\\w+)" with (?P.*|\\s)content:')) +@given(re.compile(r"File \"(?P(\.|\w)+)(?P\.\w+)\" with (?P.*|\s)content:")) def write_file_with_extras(name, extension, testdir, step, request, extra_opts, tmp_path): content = step.doc_string.content is_fixture_templated = "fixture templated" in extra_opts @@ -36,8 +36,8 @@ def write_file_with_extras(name, extension, testdir, step, request, extra_opts, format_options = dict( map(lambda fixture_name: (fixture_name, str(request.getfixturevalue(fixture_name))), template_fields) ) - makefile_arg = {name: str(content).format_map(format_options) if is_fixture_templated else content} - testdir.makefile(extension, **makefile_arg) + file_data = str(content).format_map(format_options) if is_fixture_templated else content + (Path(testdir.tmpdir.strpath) / f"{name}{extension}").write_text(file_data, encoding="utf-8") @given(