From 6771089b3f66f885307d675cdda2ecfabe6da9d8 Mon Sep 17 00:00:00 2001 From: Cedric <95940265+ddl-cedricyoung@users.noreply.github.com> Date: Wed, 26 Apr 2023 14:17:06 -0700 Subject: [PATCH] QE-11002 better reports part 2 (#325) HTML scenario report 1. add variable resolution to output ![image](https://user-images.githubusercontent.com/95940265/234357886-bf34daeb-545c-4d51-bf72-731215545d46.png) 2. add comment "step" to report ![image](https://user-images.githubusercontent.com/95940265/234358389-d75630c8-b451-4abb-ab14-beb655586450.png) 3. add Flat link ![image](https://user-images.githubusercontent.com/95940265/234357808-2eed3a4d-2a0a-402f-8127-094c8a7d041c.png) 5. disable recording env vars by default in run_details.json ![image](https://user-images.githubusercontent.com/95940265/234701469-1dccdd4c-64a6-426b-b2cf-81895b48a3f7.png) 6. use dark theme for log colors since we are using Behave's builtin colors ![image](https://user-images.githubusercontent.com/95940265/234358520-88fc5e9c-ba17-4920-bc51-e219dff973f7.png) 7. tighten up vertical spacing --- .gitignore | 1 - data/features/with_secret/cucurc.yml | 1 + .../scenario_with_comments.feature | 17 ++++++++ data/unit/ansi.log.html | 4 +- features/cli/report_basics.feature | 21 +++++++++- pyproject.toml | 2 +- src/cucu/behave_tweaks.py | 23 +++++------ src/cucu/cli/core.py | 10 +++++ src/cucu/cli/run.py | 7 +++- src/cucu/config.py | 1 + src/cucu/formatter/json.py | 39 ++++++++++++++----- src/cucu/reporter/html.py | 5 ++- src/cucu/reporter/parser.py | 7 +++- src/cucu/reporter/templates/scenario.html | 32 ++++++++++++--- src/cucu/steps/__init__.py | 1 + src/cucu/steps/comment_steps.py | 17 ++++++++ 16 files changed, 153 insertions(+), 35 deletions(-) create mode 100644 data/features/with_secret/cucurc.yml create mode 100644 data/features/with_secret/scenario_with_comments.feature create mode 100644 src/cucu/steps/comment_steps.py diff --git a/.gitignore b/.gitignore index 7f72e977..1a93ea16 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ dist .coverage.* .DS_Store *.egg-info -cucurc.yml htmlcov .nox terraform.tfstate diff --git a/data/features/with_secret/cucurc.yml b/data/features/with_secret/cucurc.yml new file mode 100644 index 00000000..d32eda46 --- /dev/null +++ b/data/features/with_secret/cucurc.yml @@ -0,0 +1 @@ +CUCU_SECRETS: MY_SECRET diff --git a/data/features/with_secret/scenario_with_comments.feature b/data/features/with_secret/scenario_with_comments.feature new file mode 100644 index 00000000..596b66aa --- /dev/null +++ b/data/features/with_secret/scenario_with_comments.feature @@ -0,0 +1,17 @@ +Feature: Feature with comments + + Scenario: Scenario with comments + * # First comment + Given I set the variable "FOO" to "bar" + And I echo "{FOO}" + Then I echo the following + """ + This is a multiline text that + can go on for a few lines + and print variables like FOO={FOO} + """ + + * # Second comment about + When I set the variable "MY_SECRET" to "buzz" + * # Comment about {MY_SECRET} + Then I echo "{MY_SECRET}" diff --git a/data/unit/ansi.log.html b/data/unit/ansi.log.html index 3719dca4..85fc3667 100644 --- a/data/unit/ansi.log.html +++ b/data/unit/ansi.log.html @@ -1,4 +1,4 @@ -
+
 
   Scenario: Just a scenario that opens a web page
     Given I start a webserver at directory "data/www" and save the port to the variable "PORT"
@@ -39,4 +39,4 @@
 Took 0m4.773s
 Error: test run failed, see above for details
 
-

+
diff --git a/features/cli/report_basics.feature b/features/cli/report_basics.feature index 7084b33b..1fa6ebe5 100644 --- a/features/cli/report_basics.feature +++ b/features/cli/report_basics.feature @@ -30,6 +30,25 @@ Feature: Report basics [\s\S]* """ + Scenario: User can run a test and see extended output + Given I run the command "cucu run data/features/with_secret/scenario_with_comments.feature --results {CUCU_RESULTS_DIR}/browser-results --env CUCU_BROKEN_IMAGES_PAGE_CHECK=disabled" and expect exit code "0" + And I run the command "cucu report {CUCU_RESULTS_DIR}/browser-results --output {CUCU_RESULTS_DIR}/browser-report" and expect exit code "0" + And I start a webserver at directory "{CUCU_RESULTS_DIR}/browser-report/" and save the port to the variable "PORT" + And I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/flat.html" + And I wait to click the link "Scenario with comments" + And I wait to click the button "show images" + + * # Can see inline comments + Then I wait to see the text "# First comment" + And I should see the text "# Second comment" + And I should see the text "# Comment about \{MY_SECRET\}" + + * # Can see variable interpolation + Then I wait to see the text "# FOO=\"bar\"" + + * # Cannot see secrets in variable interpolation + Then I wait to see the text "# MY_SECRET=\"****\"" + @QE-6852 Scenario: User can run a multi scenario test with web steps and generate report with a shareable url Given I run the command "cucu run data/features/multiple_scenarios_with_browser_steps.feature --env CUCU_BROKEN_IMAGES_PAGE_CHECK=disabled --results {CUCU_RESULTS_DIR}/multi-scenario-browser-results" and expect exit code "0" @@ -265,7 +284,7 @@ Feature: Report basics When I click the button "Feature with failing scenario with web" Then I should see a table that matches the following: | Offset | Scenario | Steps | Status | Duration | - | .* | Just a scenario that opens a web page | 3 | failed | .* | + | .* | Just a scenario that opens a web page | 3 | failed | .* | When I click the button "Just a scenario that opens a web page" And I wait to click the button "show images" And I should see the image with the alt text "And I should see the text \"inexistent\"" diff --git a/pyproject.toml b/pyproject.toml index afefec39..a1536a5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cucu" -version = "0.126.0" +version = "0.127.0" license = "MIT" description = "" authors = ["Rodney Gomes "] diff --git a/src/cucu/behave_tweaks.py b/src/cucu/behave_tweaks.py index c192cec6..42d63db7 100644 --- a/src/cucu/behave_tweaks.py +++ b/src/cucu/behave_tweaks.py @@ -126,22 +126,23 @@ def inner_step(*args, **kwargs): def hide_secrets(line): - secrets = CONFIG["CUCU_SECRETS"] + secret_keys = [ + x.strip() + for x in CONFIG.get("CUCU_SECRETS", "").split(",") + if x.strip() + ] + secret_values = [CONFIG.get(x) for x in secret_keys if CONFIG.get(x)] # here's where we can hide secrets - for secret in secrets.split(","): - if secret is not None: - secret = secret.strip() - value = CONFIG.get(secret) + for value in secret_values: - if value is not None: - replacement = "*" * len(value) + replacement = "*" * len(value) - if isinstance(line, bytes): - value = bytes(value, "utf8") - replacement = bytes(replacement, "utf8") + if isinstance(line, bytes): + value = bytes(value, "utf8") + replacement = bytes(replacement, "utf8") - line = line.replace(value, replacement) + line = line.replace(value, replacement) return line diff --git a/src/cucu/cli/core.py b/src/cucu/cli/core.py index 42615d82..496f7a0b 100644 --- a/src/cucu/cli/core.py +++ b/src/cucu/cli/core.py @@ -141,6 +141,12 @@ def main(): default=False, help="when set we will not remove any existing results directory", ) +@click.option( + "--record-env-vars", + default=False, + is_flag=True, + help="when set will record shell environment variables to debug file: run_details.json", +) @click.option( "--report", default="report", @@ -211,6 +217,7 @@ def run( logging_level, periodic_thread_dumper, preserve_results, + record_env_vars, report, report_only_failures, results, @@ -280,6 +287,9 @@ def run( if report_only_failures: os.environ["CUCU_REPORT_ONLY_FAILURES"] = "true" + if record_env_vars: + os.environ["CUCU_RECORD_ENV_VARS"] = "true" + if not dry_run: write_run_details(results, filepath) diff --git a/src/cucu/cli/run.py b/src/cucu/cli/run.py index ffb7e59d..9d9677b5 100644 --- a/src/cucu/cli/run.py +++ b/src/cucu/cli/run.py @@ -181,10 +181,15 @@ def write_run_details(results, filepath): if os.path.exists(run_details_filepath): return + if CONFIG["CUCU_RECORD_ENV_VARS"]: + env_values = dict(os.environ) + else: + env_values = "To enable use the --record-env-vars flag" + run_details = { "filepath": filepath, "full_arguments": sys.argv, - "env": dict(os.environ), + "env": env_values, "date": datetime.now().isoformat(), } diff --git a/src/cucu/config.py b/src/cucu/config.py index e2fe9eaf..16dda0a3 100644 --- a/src/cucu/config.py +++ b/src/cucu/config.py @@ -174,6 +174,7 @@ def resolve(self, string): if value is None: value = "" + # print directly to the output stream, which was taken over in behave_tweaks print(f'WARNING variable "{var_name}" is undefined') string = string.replace("{" + var_name + "}", str(value)) diff --git a/src/cucu/formatter/json.py b/src/cucu/formatter/json.py index 95bb1bd2..a579bb60 100644 --- a/src/cucu/formatter/json.py +++ b/src/cucu/formatter/json.py @@ -145,15 +145,6 @@ def match(self, match): def result(self, step): steps = self.current_feature_element["steps"] - - stdout = None - if "stdout" in step.__dict__ and step.stdout != []: - stdout = ["".join(step.stdout)] - - stderr = None - if "stderr" in step.__dict__ and step.stderr != []: - stderr = ["".join(step.stderr)] - step_index = 0 for other_step in self.steps: if other_step.unique_id == step.unique_id: @@ -167,6 +158,36 @@ def result(self, step): if step.status.name in ["passed", "failed"]: timestamp = step.start_timestamp + step_variables = CONFIG.expand(step.name) + + if step.text: + step_variables.update(CONFIG.expand(step.text)) + + if step.table: + for row in step.table.original.rows: + for value in row: + step_variables.update(CONFIG.expand(value)) + + if step_variables: + expanded = " ".join( + [ + f'{key}="{value}"' + for (key, value) in step_variables.items() + ] + ) + padding = f" {' '*(len('Given')-len(step.keyword))}" + step.stdout.insert( + 0, f"{padding}# {behave_tweaks.hide_secrets(expanded)}\n" + ) + + stdout = None + if "stdout" in step.__dict__ and step.stdout != []: + stdout = ["".join(step.stdout).rstrip()] + + stderr = None + if "stderr" in step.__dict__ and step.stderr != []: + stderr = ["".join(step.stderr).rstrip()] + steps[step_index]["result"] = { "stdout": stdout, "stderr": stderr, diff --git a/src/cucu/reporter/html.py b/src/cucu/reporter/html.py index 96dab7e9..d47f06b7 100644 --- a/src/cucu/reporter/html.py +++ b/src/cucu/reporter/html.py @@ -20,7 +20,7 @@ def escape(data): if data is None: return None - return escape_(data, {'"': """}) + return escape_(data, {'"': """}).rstrip() def process_tags(element): @@ -147,6 +147,9 @@ def generate(results, basepath, only_failures=False): ) image_filepath = os.path.join(scenario_filepath, image_filename) + if step["name"].startswith("#"): + step["heading_level"] = "h4" + if os.path.exists(image_filepath): step["image"] = urllib.parse.quote(image_filename) diff --git a/src/cucu/reporter/parser.py b/src/cucu/reporter/parser.py index 9605b8f5..3ca1cfe3 100644 --- a/src/cucu/reporter/parser.py +++ b/src/cucu/reporter/parser.py @@ -24,10 +24,13 @@ def parse_log_to_html(input: str) -> str: - """ " + """ Parse an ansi color log to html """ - result = f"
\n{REGEX.sub(lambda match: TRANSLATION[match.group(0)], html.escape(input, quote=False))}\n
\n" + + body_start = '' # use dark bg since colors are from behave + body_end = "\n" + result = f"{body_start}
\n{REGEX.sub(lambda match: TRANSLATION[match.group(0)], html.escape(input, quote=False))}\n
{body_end}" if ESC_SEQ in result: lines = "\n".join([x for x in result.split("\n") if ESC_SEQ in x]) logger.info(f"Detected unmapped ansi escape code!:\n{lines}") diff --git a/src/cucu/reporter/templates/scenario.html b/src/cucu/reporter/templates/scenario.html index 85b4abff..5fcb77b9 100644 --- a/src/cucu/reporter/templates/scenario.html +++ b/src/cucu/reporter/templates/scenario.html @@ -7,7 +7,10 @@