From 28f6070c798b435bf1c1093daea10cb87851f791 Mon Sep 17 00:00:00 2001 From: Owen Kuhn Date: Wed, 18 Dec 2024 20:22:34 -0800 Subject: [PATCH 1/4] Add karate.scenarioOutline metadata --- README.md | 3 +- .../com/intuit/karate/core/ExamplesTable.java | 20 ++++- .../com/intuit/karate/core/FeatureParser.java | 5 +- .../com/intuit/karate/core/MockHandler.java | 2 +- .../java/com/intuit/karate/core/Scenario.java | 21 ++++- .../intuit/karate/core/ScenarioBridge.java | 4 + .../intuit/karate/core/ScenarioOutline.java | 25 +++++- .../karate/core/ScenarioOutlineResult.java | 78 +++++++++++++++++++ .../intuit/karate/core/ScenarioResult.java | 30 ++++++- .../intuit/karate/core/ScenarioRuntime.java | 3 + .../java/com/intuit/karate/core/Table.java | 14 ++++ .../core/ScenarioOutlineResultTest.feature | 29 +++++++ .../karate/core/feature-result-cucumber.json | 64 +++++++++++++++ .../intuit/karate/core/feature-result.feature | 4 + .../intuit/karate/core/feature-result.json | 65 +++++++++++++++- .../core/scenario-outline-result.feature | 58 ++++++++++++++ 16 files changed, 409 insertions(+), 16 deletions(-) create mode 100644 karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java create mode 100644 karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.feature create mode 100644 karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature diff --git a/README.md b/README.md index b160ff88c..86055d690 100755 --- a/README.md +++ b/README.md @@ -3690,6 +3690,7 @@ Operation | Description karate.response | returns the last HTTP response as a JS object that enables advanced use-cases such as getting a header ignoring case: `karate.response.header('some-header')` karate.request | returns the last HTTP request as a JS object that enables advanced use-cases such as getting a header ignoring case: `karate.request.header('some-header')`, which works [even in mocks](https://github.com/karatelabs/karate/tree/master/karate-netty#requestheaders) karate.scenario | get metadata about the currently executing `Scenario` (or `Outline` - `Example`) within a test +karate.scenarioOutline | get metadata about the currently executing scenario outline within a test karate.set(name, value) | sets the value of a variable (immediately), which may be needed in case any other routines (such as the [configured headers](#configure-headers)) depend on that variable karate.set(object) | where the single argument is expected to be a `Map` or JSON-like, and will perform the above `karate.set()` operation for all key-value pairs in one-shot karate.set(name, path, value) | only needed when you need to conditionally build payload elements, especially XML. This is best explained via [an example](karate-core/src/test/java/com/intuit/karate/core/xml/xml.feature#L211), and it behaves the same way as the [`set`](#set) keyword. Also see [`eval`](#eval). @@ -4530,7 +4531,7 @@ This is great for testing boundary conditions against a single end-point, with t ### Scenario Outline Enhancements Karate has enhanced the Cucumber `Scenario Outline` as follows: * __Type Hints__: if the `Examples` column header has a `!` appended, each value will be evaluated as a JavaScript data-type (number, boolean, or *even* in-line JSON) - else it defaults to string. -* __Magic Variables__: `__row` gives you the entire row as a JSON object, and `__num` gives you the row index (the first row is `0`). +* __Magic Variables__: `__row` gives you the entire row as a JSON object, and `__num` gives you the row index (the first row is `0`). In rare cases you may have multiple `Examples` tables for a single outline, `__tableNum` will give you the index of the currently executing table. * __Auto Variables__: in addition to `__row`, each column key-value will be available as a separate [variable](#def), which greatly simplifies JSON manipulation - especially when you want to re-use JSON [files](#reading-files) containing [embedded expressions](#embedded-expressions). * Any empty cells will result in a `null` value for that column-key, and this can be useful to [remove nodes](#remove-if-null) from JSON or XML documents diff --git a/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java b/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java index 14cdb90bf..1aef761ee 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java @@ -24,7 +24,9 @@ package com.intuit.karate.core; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * @@ -34,11 +36,14 @@ public class ExamplesTable { private final ScenarioOutline outline; private final Table table; + private final int index; private List tags; + - public ExamplesTable(ScenarioOutline outline, Table table) { + public ExamplesTable(ScenarioOutline outline, Table table, int index) { this.outline = outline; this.table = table; + this.index = index; this.tags = new ArrayList(); } @@ -58,4 +63,17 @@ public Table getTable() { return table; } + public int getIndex() { + return index; + } + + public Map toKarateJson() { + Map map = new HashMap(); + List tagStrings = new ArrayList(); + tags.forEach(tag -> tagStrings.add(tag.toString())); + map.put("tags", tagStrings); + map.put("data", table.getRowsAsMapsConverted()); + return map; + } + } diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java index 5cf3a2ddc..cbfc8745f 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java @@ -240,7 +240,7 @@ public void enterBackground(KarateParser.BackgroundContext ctx) { @Override public void enterScenario(KarateParser.ScenarioContext ctx) { FeatureSection section = new FeatureSection(); - Scenario scenario = new Scenario(feature, section, -1); + Scenario scenario = new Scenario(feature, section, -1, -1); scenario.setLine(getActualLine(ctx.SCENARIO())); section.setScenario(scenario); feature.addSection(section); @@ -279,9 +279,10 @@ public void enterScenarioOutline(KarateParser.ScenarioOutlineContext ctx) { } List examples = new ArrayList(ctx.examples().size()); outline.setExamplesTables(examples); + int tableIndex = 0; for (KarateParser.ExamplesContext ec : ctx.examples()) { Table table = toTable(ec.table()); - ExamplesTable example = new ExamplesTable(outline, table); + ExamplesTable example = new ExamplesTable(outline, table, tableIndex++); examples.add(example); if (ec.tags() != null) { example.setTags(toTags(-1, ec.tags().TAGS())); diff --git a/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java b/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java index 829194edc..d491e35db 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java +++ b/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java @@ -117,7 +117,7 @@ private ScenarioRuntime initRuntime(Feature feature, Map args) { FeatureRuntime featureRuntime = FeatureRuntime.of(Suite.forTempUse(HttpClientFactory.DEFAULT), new FeatureCall(feature), args); FeatureSection section = new FeatureSection(); section.setIndex(-1); // TODO util for creating dummy scenario - Scenario dummy = new Scenario(feature, section, -1); + Scenario dummy = new Scenario(feature, section, -1, -1); section.setScenario(dummy); ScenarioRuntime runtime = new ScenarioRuntime(featureRuntime, dummy); runtime.logger.setLogOnly(true); diff --git a/karate-core/src/main/java/com/intuit/karate/core/Scenario.java b/karate-core/src/main/java/com/intuit/karate/core/Scenario.java index 25edc3eb0..5c72a4d45 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/Scenario.java +++ b/karate-core/src/main/java/com/intuit/karate/core/Scenario.java @@ -36,6 +36,7 @@ public class Scenario { private final Feature feature; private final FeatureSection section; + private final int exampleTableIndex; private final int exampleIndex; private int line; @@ -46,9 +47,10 @@ public class Scenario { private Map exampleData; private String dynamicExpression; - public Scenario(Feature feature, FeatureSection section, int exampleIndex) { + public Scenario(Feature feature, FeatureSection section, int exampleTableIndex, int exampleIndex) { this.feature = feature; this.section = section; + this.exampleTableIndex = exampleTableIndex; this.exampleIndex = exampleIndex; } @@ -80,7 +82,7 @@ public String getRefIdAndName() { // only called for dynamic scenarios public Scenario copy(int exampleIndex) { - Scenario s = new Scenario(feature, section, exampleIndex); + Scenario s = new Scenario(feature, section, exampleTableIndex, exampleIndex); s.name = name; s.description = description; s.tags = tags; @@ -134,6 +136,9 @@ public Step getStepByLine(int line) { public String getRefId() { int num = section.getIndex() + 1; String meta = "[" + num; + if (exampleTableIndex != -1) { + meta = meta + "." + (exampleTableIndex + 1); + } if (exampleIndex != -1) { meta = meta + "." + (exampleIndex + 1); } @@ -146,7 +151,13 @@ public String getDebugInfo() { public String getUniqueId() { String id = feature.getResource().getPackageQualifiedName() + "_" + (section.getIndex() + 1); - return exampleIndex == -1 ? id : id + "_" + (exampleIndex + 1); + if (exampleTableIndex != -1) { + id += "_" + (exampleTableIndex + 1); + } + if (exampleIndex != -1) { + id += "_" + (exampleIndex + 1); + } + return id; } public List getBackgroundSteps() { @@ -245,6 +256,10 @@ public void setExampleData(Map exampleData) { this.exampleData = exampleData; } + public int getExampleTableIndex() { + return exampleTableIndex; + } + public int getExampleIndex() { return exampleIndex; } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioBridge.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioBridge.java index 19d3a4dd6..f793d5cf7 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioBridge.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioBridge.java @@ -551,6 +551,10 @@ public Object getScenario() { return new JsMap(getEngine().runtime.result.toKarateJson()); } + public Object getScenarioOutline() { + return new JsMap(getEngine().runtime.outlineResult.toKarateJson()); + } + public Object getTags() { return JsValue.fromJava(getEngine().runtime.tags.getTags()); } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java index fb5e3ac5d..c74834d8b 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * @author pthomas3 @@ -40,14 +41,15 @@ public class ScenarioOutline { private String description; private List steps; private List examplesTables; + private int numScenarios = 0; public ScenarioOutline(Feature feature, FeatureSection section) { this.feature = feature; this.section = section; } - public Scenario toScenario(String dynamicExpression, int exampleIndex, int updateLine, List tagsForExamples) { - Scenario s = new Scenario(feature, section, exampleIndex); + public Scenario toScenario(String dynamicExpression, int exampleTableIndex, int exampleIndex, int updateLine, List tagsForExamples) { + Scenario s = new Scenario(feature, section, exampleTableIndex, exampleIndex); s.setName(name); s.setDescription(description); s.setLine(updateLine); @@ -75,6 +77,7 @@ public Scenario toScenario(String dynamicExpression, int exampleIndex, int updat step.setTable(original.getTable()); step.setComments(original.getComments()); } + numScenarios++; return s; } @@ -101,13 +104,13 @@ public List getScenarios(FeatureRuntime fr) { if (selectedForExecution) { Table table = examples.getTable(); if (table.isDynamic()) { - Scenario scenario = toScenario(table.getDynamicExpression(), -1, table.getLineNumberForRow(0), examples.getTags()); + Scenario scenario = toScenario(table.getDynamicExpression(), examples.getIndex(), -1, table.getLineNumberForRow(0), examples.getTags()); list.add(scenario); } else { int rowCount = table.getRows().size(); for (int i = 1; i < rowCount; i++) { // don't include header row int exampleIndex = i - 1; // next line will set exampleIndex on scenario - Scenario scenario = toScenario(null, exampleIndex, table.getLineNumberForRow(i), examples.getTags()); + Scenario scenario = toScenario(null, examples.getIndex(), exampleIndex, table.getLineNumberForRow(i), examples.getTags()); scenario.setExampleData(table.getExampleData(exampleIndex)); // and we set exampleData here list.add(scenario); for (String key : table.getKeys()) { @@ -167,9 +170,23 @@ public void setSteps(List steps) { public List getExamplesTables() { return examplesTables; } + + public int getNumExampleTables() { + return examplesTables.size(); + } + + public List> getAllExampleData() { + List> exampleData = new ArrayList(); + examplesTables.forEach(table -> exampleData.add(table.toKarateJson())); + return exampleData; + } public void setExamplesTables(List examplesTables) { this.examplesTables = examplesTables; } + + public int getNumScenarios() { + return numScenarios; + } } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java new file mode 100644 index 000000000..c8b16c8f1 --- /dev/null +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java @@ -0,0 +1,78 @@ +/* + * The MIT License + * + * Copyright 2022 Karate Labs Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.core; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * @author OwenK2 + */ +public class ScenarioOutlineResult { + + final private ScenarioOutline scenarioOutline; + final private ScenarioRuntime runtime; + + public ScenarioOutlineResult(ScenarioOutline scenarioOutline, ScenarioRuntime runtime) { + // NOTE: this value can be null, in which case the scenario is not from an outline + this.scenarioOutline = scenarioOutline; + this.runtime = runtime; + } + + public Map toKarateJson() { + if (scenarioOutline == null) return null; + Map map = new HashMap(); + map.put("name", scenarioOutline.getName()); + map.put("description", scenarioOutline.getDescription()); + map.put("line", scenarioOutline.getLine()); + map.put("sectionIndex", scenarioOutline.getSection().getIndex()); + map.put("exampleTableCount", scenarioOutline.getNumExampleTables()); + map.put("exampleTables", scenarioOutline.getAllExampleData()); + map.put("numScenariosToExecute", scenarioOutline.getNumScenarios()); + + // Get results of other examples in this outline + List> scenarioResults = new ArrayList(); + if (runtime.featureRuntime != null && runtime.featureRuntime.result != null) { + // Add all past results + runtime.featureRuntime.result.getScenarioResults().forEach(result -> { + if (result.getScenario().getSection().getIndex() == scenarioOutline.getSection().getIndex()) { + scenarioResults.add(result.toInfoJson()); + } + }); + + // Add most recent result + if (runtime.result != null) { + scenarioResults.add(runtime.result.toInfoJson()); + } + } + map.put("scenarioResults", scenarioResults); + map.put("numScenariosExecuted", scenarioResults.size()); + + return map; + } + +} diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java index 993496caa..af6ad5007 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java @@ -58,6 +58,10 @@ public int compareTo(ScenarioResult sr) { if (delta != 0) { return delta; } + delta = scenario.getExampleTableIndex() - sr.scenario.getExampleTableIndex(); + if (delta != 0) { + return delta; + } return scenario.getExampleIndex() - sr.scenario.getExampleIndex(); } @@ -137,9 +141,10 @@ private List getStepResults(boolean background) { public static ScenarioResult fromKarateJson(File workingDir, Feature feature, Map map) { int sectionIndex = (Integer) map.get("sectionIndex"); + int exampleTableIndex = (Integer) map.get("exampleTableIndex"); int exampleIndex = (Integer) map.get("exampleIndex"); FeatureSection section = feature.getSection(sectionIndex); - Scenario scenario = new Scenario(feature, section, exampleIndex); + Scenario scenario = new Scenario(feature, section, exampleTableIndex, exampleIndex); if (section.isOutline()) { scenario.setTags(section.getScenarioOutline().getTags()); scenario.setDescription(section.getScenarioOutline().getDescription()); @@ -194,6 +199,7 @@ public Map toKarateJson() { } //====================================================================== map.put("sectionIndex", scenario.getSection().getIndex()); + map.put("exampleTableIndex", scenario.getExampleTableIndex()); map.put("exampleIndex", scenario.getExampleIndex()); Map exampleData = scenario.getExampleData(); if (exampleData != null) { @@ -213,6 +219,28 @@ public Map toKarateJson() { return map; } + // Paired down information for use in karate.scenarioOutline + public Map toInfoJson() { + Map map = new HashMap(); + map.put("durationMillis", getDurationMillis()); + List tags = scenario.getTagsEffective().getTags(); + if (tags != null && !tags.isEmpty()) { + map.put("tags", tags); + } + map.put("failed", isFailed()); + map.put("refId", scenario.getRefId()); + map.put("sectionIndex", scenario.getSection().getIndex()); + map.put("exampleTableIndex", scenario.getExampleTableIndex()); + map.put("exampleIndex", scenario.getExampleIndex()); + map.put("name", scenario.getName()); + map.put("description", scenario.getDescription()); + map.put("line", scenario.getLine()); + map.put("executorName", executorName); + map.put("startTime", startTime); + map.put("endTime", endTime); + return map; + } + public Map toCucumberJson() { Map map = new HashMap(); map.put("name", scenario.getName()); diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java index 407dda37f..87332e0a2 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java @@ -53,6 +53,7 @@ public class ScenarioRuntime implements Runnable { public final Tags tags; public final ScenarioActions actions; public final ScenarioResult result; + public final ScenarioOutlineResult outlineResult; public final ScenarioEngine engine; public final boolean reportDisabled; public final Map magicVariables; @@ -91,6 +92,7 @@ public ScenarioRuntime(FeatureRuntime featureRuntime, Scenario scenario) { magicVariables = initMagicVariables(); } result = new ScenarioResult(scenario); + outlineResult = new ScenarioOutlineResult(scenario.getSection().getScenarioOutline(), this); if (featureRuntime.setupResult != null) { // TODO improve this and simplify report rendering code in report/karate-feature.html StepResult sr = result.addFakeStepResult("@setup", null); @@ -132,6 +134,7 @@ private Map initMagicVariables() { map.putAll(exampleData); map.put("__row", exampleData); map.put("__num", scenario.getExampleIndex()); + map.put("__tableNum", scenario.getExampleTableIndex()); } return map; } diff --git a/karate-core/src/main/java/com/intuit/karate/core/Table.java b/karate-core/src/main/java/com/intuit/karate/core/Table.java index 0d2c543d8..85375a13c 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/Table.java +++ b/karate-core/src/main/java/com/intuit/karate/core/Table.java @@ -175,6 +175,20 @@ public List> getRowsAsMaps() { return list; } + public List> getRowsAsMapsConverted() { + int rowCount = rows.size(); + List> list = new ArrayList(rowCount - 1); + for (int i = 1; i < rowCount; i++) { // don't include header row + Map map = new LinkedHashMap(cols.size()); + list.add(map); + List row = rows.get(i); + for (Column col : cols) { + map.put(col.key, convert(row.get(col.index), col)); + } + } + return list; + } + private static Object convert(String raw, Column col) { try { switch (col.type) { diff --git a/karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.feature b/karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.feature new file mode 100644 index 000000000..8119f67bb --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.feature @@ -0,0 +1,29 @@ +package com.intuit.karate.core; + +import com.intuit.karate.FileUtils; +import com.intuit.karate.TestUtils; +import static com.intuit.karate.TestUtils.*; +import com.intuit.karate.report.Report; +import com.intuit.karate.report.SuiteReports; +import java.io.File; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author OwenK2 + */ +class ScenarioOutlineResultTest { + + static final Logger logger = LoggerFactory.getLogger(ScenarioOutlineResultTest.class); + + @Test + void testJsonConversion() { + FeatureRuntime fr = TestUtils.runFeature("classpath:com/intuit/karate/core/scenario-outline-result.feature"); + assertFalse(fr.result.isFailed()); + } + +} diff --git a/karate-core/src/test/java/com/intuit/karate/core/feature-result-cucumber.json b/karate-core/src/test/java/com/intuit/karate/core/feature-result-cucumber.json index fcd25f4de..7be40d484 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/feature-result-cucumber.json +++ b/karate-core/src/test/java/com/intuit/karate/core/feature-result-cucumber.json @@ -305,6 +305,70 @@ "line": 1 } ] + }, + { + "line": 5, + "name": "", + "description": "", + "type": "background", + "keyword": "Background", + "steps": [ + { + "name": "print 'in background'", + "result": { + "duration": "#number", + "status": "passed" + }, + "match": { + "location": "karate", + "arguments": [ + ] + }, + "keyword": "*", + "line": 6, + "doc_string": { + "content_type": "", + "value": "#string", + "line": 6 + } + } + ] + }, + { + "line": 26, + "name": "hello baz", + "description": "", + "id": "hello-baz", + "type": "scenario", + "start_timestamp": "#string", + "keyword": "Scenario Outline", + "steps": [ + { + "name": "print 'name:', name", + "result": { + "duration": "#number", + "status": "passed" + }, + "match": { + "location": "karate", + "arguments": [ + ] + }, + "keyword": "*", + "line": 17, + "doc_string": { + "content_type": "", + "value": "#string", + "line": 17 + } + } + ], + "tags": [ + { + "name": "@one", + "line": 1 + } + ] } ], "name": "com\/intuit\/karate\/core\/feature-result.feature", diff --git a/karate-core/src/test/java/com/intuit/karate/core/feature-result.feature b/karate-core/src/test/java/com/intuit/karate/core/feature-result.feature index 74798aeda..cc6db4cda 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/feature-result.feature +++ b/karate-core/src/test/java/com/intuit/karate/core/feature-result.feature @@ -20,3 +20,7 @@ Examples: | name | | foo | | bar | + +Examples: +| name | +| baz | \ No newline at end of file diff --git a/karate-core/src/test/java/com/intuit/karate/core/feature-result.json b/karate-core/src/test/java/com/intuit/karate/core/feature-result.json index 863ff5cad..6e2637d56 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/feature-result.json +++ b/karate-core/src/test/java/com/intuit/karate/core/feature-result.json @@ -121,6 +121,7 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": -1, + "exampleTableIndex": -1, "refId": "[1:3]", "durationMillis": "#number", "failed": false @@ -165,6 +166,7 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": -1, + "exampleTableIndex": -1, "refId": "[1:9]", "durationMillis": "#number", "failed": false, @@ -217,7 +219,8 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": 0, - "refId": "[2.1:21]", + "exampleTableIndex": 0, + "refId": "[2.1.1:21]", "durationMillis": "#number", "failed": false, "tags": [ @@ -271,7 +274,8 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": 1, - "refId": "[2.2:22]", + "exampleTableIndex": 0, + "refId": "[2.1.2:22]", "durationMillis": "#number", "failed": false, "tags": [ @@ -280,6 +284,61 @@ "exampleData": { "name": "bar" } + }, + { + "stepResults": [ + { + "result": { + "nanos": "#number", + "millis": "#number", + "status": "passed", + "startTime": "#number", + "endTime": "#number" + }, + "step": { + "background": true, + "line": 6, + "prefix": "*", + "index": 0, + "text": "print 'in background'" + }, + "stepLog": "#string" + }, + { + "result": { + "nanos": "#number", + "millis": "#number", + "status": "passed", + "startTime": "#number", + "endTime": "#number" + }, + "step": { + "line": 17, + "prefix": "*", + "index": 0, + "text": "print 'name:', name" + }, + "stepLog": "#string" + } + ], + "executorName": "#string", + "name": "hello baz", + "description": "", + "line": 26, + "sectionIndex": 1, + "startTime": "#number", + "endTime": "#number", + "exampleIndex": 0, + "exampleTableIndex": 1, + "refId": "[2.2.1:26]", + "durationMillis": "#number", + "failed": false, + "tags": [ + "one" + ], + "exampleData": { + "name": "baz" + } } ], "prefixedPath": "classpath:com\/intuit\/karate\/core\/feature-result.feature", @@ -289,7 +348,7 @@ "description": "my description", "durationMillis": "#number", "resultDate": "#string", - "passedCount": 3, + "passedCount": 4, "failedCount": 0, "callDepth": 0, "loopIndex": -1 diff --git a/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature b/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature new file mode 100644 index 000000000..b6314decd --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature @@ -0,0 +1,58 @@ +Feature: my feature + +Scenario: single scenario + * match karate.scenarioOutline == null + +Scenario Outline: outline + description of outline + + # Confirm table index + * match __tableNum == + * match karate.scenario.exampleTableIndex ==
+ + # Confirm example index within table + * match __num == + * match karate.scenario.exampleIndex == + + # Confirm scenarioOutline result + * match karate.scenarioOutline == + """ + { + "sectionIndex": 1, + "numScenariosToExecute": 4, + "exampleTableCount": 2, + "line": 6, + "name": "outline", + "numScenariosExecuted": , + "description": "description of outline", + "exampleTables": [ + { + "data": [ + {table: 0, num: 0, executedSoFar: 1}, + {table: 0, num: 1, executedSoFar: 2}, + ], + "tags": ["@one", "@two"] + }, + { + "data": [ + {table: 1, num: 0, executedSoFar: 3}, + {table: 1, num: 1, executedSoFar: 4}, + ], + "tags": ["@three", "@four"] + }, + ], + "scenarioResults": "#[] #object" + } + """ + + @one @two + Examples: + | table! | num! | executedSoFar! | + | 0 | 0 | 1 | + | 0 | 1 | 2 | + + @three @four + Examples: + | table! | num! | executedSoFar! | + | 1 | 0 | 3 | + | 1 | 1 | 4 | \ No newline at end of file From 6e84b5dba4486382422273dc5fdacaec4548aa2f Mon Sep 17 00:00:00 2001 From: Owen Kuhn Date: Thu, 19 Dec 2024 11:25:21 -0800 Subject: [PATCH 2/4] Remove table index --- .../com/intuit/karate/core/ExamplesTable.java | 8 +------ .../com/intuit/karate/core/FeatureParser.java | 5 ++-- .../com/intuit/karate/core/MockHandler.java | 2 +- .../java/com/intuit/karate/core/Scenario.java | 21 +++------------- .../intuit/karate/core/ScenarioOutline.java | 8 +++---- .../intuit/karate/core/ScenarioResult.java | 9 +------ .../intuit/karate/core/ScenarioRuntime.java | 1 - .../intuit/karate/core/feature-result.json | 11 +++------ .../core/scenario-outline-result.feature | 24 ++++++++----------- 9 files changed, 25 insertions(+), 64 deletions(-) diff --git a/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java b/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java index 1aef761ee..92e97c58e 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java @@ -36,14 +36,12 @@ public class ExamplesTable { private final ScenarioOutline outline; private final Table table; - private final int index; private List tags; - public ExamplesTable(ScenarioOutline outline, Table table, int index) { + public ExamplesTable(ScenarioOutline outline, Table table) { this.outline = outline; this.table = table; - this.index = index; this.tags = new ArrayList(); } @@ -63,10 +61,6 @@ public Table getTable() { return table; } - public int getIndex() { - return index; - } - public Map toKarateJson() { Map map = new HashMap(); List tagStrings = new ArrayList(); diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java index cbfc8745f..5cf3a2ddc 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java @@ -240,7 +240,7 @@ public void enterBackground(KarateParser.BackgroundContext ctx) { @Override public void enterScenario(KarateParser.ScenarioContext ctx) { FeatureSection section = new FeatureSection(); - Scenario scenario = new Scenario(feature, section, -1, -1); + Scenario scenario = new Scenario(feature, section, -1); scenario.setLine(getActualLine(ctx.SCENARIO())); section.setScenario(scenario); feature.addSection(section); @@ -279,10 +279,9 @@ public void enterScenarioOutline(KarateParser.ScenarioOutlineContext ctx) { } List examples = new ArrayList(ctx.examples().size()); outline.setExamplesTables(examples); - int tableIndex = 0; for (KarateParser.ExamplesContext ec : ctx.examples()) { Table table = toTable(ec.table()); - ExamplesTable example = new ExamplesTable(outline, table, tableIndex++); + ExamplesTable example = new ExamplesTable(outline, table); examples.add(example); if (ec.tags() != null) { example.setTags(toTags(-1, ec.tags().TAGS())); diff --git a/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java b/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java index d491e35db..829194edc 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java +++ b/karate-core/src/main/java/com/intuit/karate/core/MockHandler.java @@ -117,7 +117,7 @@ private ScenarioRuntime initRuntime(Feature feature, Map args) { FeatureRuntime featureRuntime = FeatureRuntime.of(Suite.forTempUse(HttpClientFactory.DEFAULT), new FeatureCall(feature), args); FeatureSection section = new FeatureSection(); section.setIndex(-1); // TODO util for creating dummy scenario - Scenario dummy = new Scenario(feature, section, -1, -1); + Scenario dummy = new Scenario(feature, section, -1); section.setScenario(dummy); ScenarioRuntime runtime = new ScenarioRuntime(featureRuntime, dummy); runtime.logger.setLogOnly(true); diff --git a/karate-core/src/main/java/com/intuit/karate/core/Scenario.java b/karate-core/src/main/java/com/intuit/karate/core/Scenario.java index 5c72a4d45..25edc3eb0 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/Scenario.java +++ b/karate-core/src/main/java/com/intuit/karate/core/Scenario.java @@ -36,7 +36,6 @@ public class Scenario { private final Feature feature; private final FeatureSection section; - private final int exampleTableIndex; private final int exampleIndex; private int line; @@ -47,10 +46,9 @@ public class Scenario { private Map exampleData; private String dynamicExpression; - public Scenario(Feature feature, FeatureSection section, int exampleTableIndex, int exampleIndex) { + public Scenario(Feature feature, FeatureSection section, int exampleIndex) { this.feature = feature; this.section = section; - this.exampleTableIndex = exampleTableIndex; this.exampleIndex = exampleIndex; } @@ -82,7 +80,7 @@ public String getRefIdAndName() { // only called for dynamic scenarios public Scenario copy(int exampleIndex) { - Scenario s = new Scenario(feature, section, exampleTableIndex, exampleIndex); + Scenario s = new Scenario(feature, section, exampleIndex); s.name = name; s.description = description; s.tags = tags; @@ -136,9 +134,6 @@ public Step getStepByLine(int line) { public String getRefId() { int num = section.getIndex() + 1; String meta = "[" + num; - if (exampleTableIndex != -1) { - meta = meta + "." + (exampleTableIndex + 1); - } if (exampleIndex != -1) { meta = meta + "." + (exampleIndex + 1); } @@ -151,13 +146,7 @@ public String getDebugInfo() { public String getUniqueId() { String id = feature.getResource().getPackageQualifiedName() + "_" + (section.getIndex() + 1); - if (exampleTableIndex != -1) { - id += "_" + (exampleTableIndex + 1); - } - if (exampleIndex != -1) { - id += "_" + (exampleIndex + 1); - } - return id; + return exampleIndex == -1 ? id : id + "_" + (exampleIndex + 1); } public List getBackgroundSteps() { @@ -256,10 +245,6 @@ public void setExampleData(Map exampleData) { this.exampleData = exampleData; } - public int getExampleTableIndex() { - return exampleTableIndex; - } - public int getExampleIndex() { return exampleIndex; } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java index c74834d8b..cfe376090 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java @@ -48,8 +48,8 @@ public ScenarioOutline(Feature feature, FeatureSection section) { this.section = section; } - public Scenario toScenario(String dynamicExpression, int exampleTableIndex, int exampleIndex, int updateLine, List tagsForExamples) { - Scenario s = new Scenario(feature, section, exampleTableIndex, exampleIndex); + public Scenario toScenario(String dynamicExpression, int exampleIndex, int updateLine, List tagsForExamples) { + Scenario s = new Scenario(feature, section, exampleIndex); s.setName(name); s.setDescription(description); s.setLine(updateLine); @@ -104,13 +104,13 @@ public List getScenarios(FeatureRuntime fr) { if (selectedForExecution) { Table table = examples.getTable(); if (table.isDynamic()) { - Scenario scenario = toScenario(table.getDynamicExpression(), examples.getIndex(), -1, table.getLineNumberForRow(0), examples.getTags()); + Scenario scenario = toScenario(table.getDynamicExpression(), -1, table.getLineNumberForRow(0), examples.getTags()); list.add(scenario); } else { int rowCount = table.getRows().size(); for (int i = 1; i < rowCount; i++) { // don't include header row int exampleIndex = i - 1; // next line will set exampleIndex on scenario - Scenario scenario = toScenario(null, examples.getIndex(), exampleIndex, table.getLineNumberForRow(i), examples.getTags()); + Scenario scenario = toScenario(null, exampleIndex, table.getLineNumberForRow(i), examples.getTags()); scenario.setExampleData(table.getExampleData(exampleIndex)); // and we set exampleData here list.add(scenario); for (String key : table.getKeys()) { diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java index af6ad5007..1786374b5 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java @@ -58,10 +58,6 @@ public int compareTo(ScenarioResult sr) { if (delta != 0) { return delta; } - delta = scenario.getExampleTableIndex() - sr.scenario.getExampleTableIndex(); - if (delta != 0) { - return delta; - } return scenario.getExampleIndex() - sr.scenario.getExampleIndex(); } @@ -141,10 +137,9 @@ private List getStepResults(boolean background) { public static ScenarioResult fromKarateJson(File workingDir, Feature feature, Map map) { int sectionIndex = (Integer) map.get("sectionIndex"); - int exampleTableIndex = (Integer) map.get("exampleTableIndex"); int exampleIndex = (Integer) map.get("exampleIndex"); FeatureSection section = feature.getSection(sectionIndex); - Scenario scenario = new Scenario(feature, section, exampleTableIndex, exampleIndex); + Scenario scenario = new Scenario(feature, section, exampleIndex); if (section.isOutline()) { scenario.setTags(section.getScenarioOutline().getTags()); scenario.setDescription(section.getScenarioOutline().getDescription()); @@ -199,7 +194,6 @@ public Map toKarateJson() { } //====================================================================== map.put("sectionIndex", scenario.getSection().getIndex()); - map.put("exampleTableIndex", scenario.getExampleTableIndex()); map.put("exampleIndex", scenario.getExampleIndex()); Map exampleData = scenario.getExampleData(); if (exampleData != null) { @@ -230,7 +224,6 @@ public Map toInfoJson() { map.put("failed", isFailed()); map.put("refId", scenario.getRefId()); map.put("sectionIndex", scenario.getSection().getIndex()); - map.put("exampleTableIndex", scenario.getExampleTableIndex()); map.put("exampleIndex", scenario.getExampleIndex()); map.put("name", scenario.getName()); map.put("description", scenario.getDescription()); diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java index 87332e0a2..7c290d8ed 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java @@ -134,7 +134,6 @@ private Map initMagicVariables() { map.putAll(exampleData); map.put("__row", exampleData); map.put("__num", scenario.getExampleIndex()); - map.put("__tableNum", scenario.getExampleTableIndex()); } return map; } diff --git a/karate-core/src/test/java/com/intuit/karate/core/feature-result.json b/karate-core/src/test/java/com/intuit/karate/core/feature-result.json index 6e2637d56..ff8e81259 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/feature-result.json +++ b/karate-core/src/test/java/com/intuit/karate/core/feature-result.json @@ -121,7 +121,6 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": -1, - "exampleTableIndex": -1, "refId": "[1:3]", "durationMillis": "#number", "failed": false @@ -166,7 +165,6 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": -1, - "exampleTableIndex": -1, "refId": "[1:9]", "durationMillis": "#number", "failed": false, @@ -219,8 +217,7 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": 0, - "exampleTableIndex": 0, - "refId": "[2.1.1:21]", + "refId": "[2.1:21]", "durationMillis": "#number", "failed": false, "tags": [ @@ -274,8 +271,7 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": 1, - "exampleTableIndex": 0, - "refId": "[2.1.2:22]", + "refId": "[2.2:22]", "durationMillis": "#number", "failed": false, "tags": [ @@ -329,8 +325,7 @@ "startTime": "#number", "endTime": "#number", "exampleIndex": 0, - "exampleTableIndex": 1, - "refId": "[2.2.1:26]", + "refId": "[2.1:26]", "durationMillis": "#number", "failed": false, "tags": [ diff --git a/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature b/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature index b6314decd..72c2e05d6 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature +++ b/karate-core/src/test/java/com/intuit/karate/core/scenario-outline-result.feature @@ -6,10 +6,6 @@ Scenario: single scenario Scenario Outline: outline description of outline - # Confirm table index - * match __tableNum ==
- * match karate.scenario.exampleTableIndex ==
- # Confirm example index within table * match __num == * match karate.scenario.exampleIndex == @@ -28,15 +24,15 @@ Scenario Outline: outline "exampleTables": [ { "data": [ - {table: 0, num: 0, executedSoFar: 1}, - {table: 0, num: 1, executedSoFar: 2}, + {num: 0, executedSoFar: 1}, + {num: 1, executedSoFar: 2}, ], "tags": ["@one", "@two"] }, { "data": [ - {table: 1, num: 0, executedSoFar: 3}, - {table: 1, num: 1, executedSoFar: 4}, + {num: 0, executedSoFar: 3}, + {num: 1, executedSoFar: 4}, ], "tags": ["@three", "@four"] }, @@ -47,12 +43,12 @@ Scenario Outline: outline @one @two Examples: - | table! | num! | executedSoFar! | - | 0 | 0 | 1 | - | 0 | 1 | 2 | + | num! | executedSoFar! | + | 0 | 1 | + | 1 | 2 | @three @four Examples: - | table! | num! | executedSoFar! | - | 1 | 0 | 3 | - | 1 | 1 | 4 | \ No newline at end of file + | num! | executedSoFar! | + | 0 | 3 | + | 1 | 4 | \ No newline at end of file From c79e31b4267c369b7de42cc2248712c907684473 Mon Sep 17 00:00:00 2001 From: Owen Kuhn Date: Thu, 19 Dec 2024 14:57:44 -0800 Subject: [PATCH 3/4] Forgot to remove magic var from reamde --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86055d690..4052be1ed 100755 --- a/README.md +++ b/README.md @@ -4531,7 +4531,7 @@ This is great for testing boundary conditions against a single end-point, with t ### Scenario Outline Enhancements Karate has enhanced the Cucumber `Scenario Outline` as follows: * __Type Hints__: if the `Examples` column header has a `!` appended, each value will be evaluated as a JavaScript data-type (number, boolean, or *even* in-line JSON) - else it defaults to string. -* __Magic Variables__: `__row` gives you the entire row as a JSON object, and `__num` gives you the row index (the first row is `0`). In rare cases you may have multiple `Examples` tables for a single outline, `__tableNum` will give you the index of the currently executing table. +* __Magic Variables__: `__row` gives you the entire row as a JSON object, and `__num` gives you the row index (the first row is `0`). * __Auto Variables__: in addition to `__row`, each column key-value will be available as a separate [variable](#def), which greatly simplifies JSON manipulation - especially when you want to re-use JSON [files](#reading-files) containing [embedded expressions](#embedded-expressions). * Any empty cells will result in a `null` value for that column-key, and this can be useful to [remove nodes](#remove-if-null) from JSON or XML documents From 0da7b5657414afc091c05746750b5ad360f0f540 Mon Sep 17 00:00:00 2001 From: OwenK2 Date: Tue, 24 Dec 2024 15:15:13 -0800 Subject: [PATCH 4/4] Add afterScenarioOutline hook --- README.md | 2 + .../java/com/intuit/karate/RuntimeHook.java | 4 ++ .../com/intuit/karate/core/AfterHookType.java | 45 +++++++++++++++++++ .../java/com/intuit/karate/core/Config.java | 13 ++++++ .../intuit/karate/core/FeatureRuntime.java | 24 +++++++++- .../intuit/karate/core/ScenarioEngine.java | 25 ++++++++--- .../karate/core/ScenarioOutlineResult.java | 12 +++-- .../intuit/karate/core/ScenarioResult.java | 14 ++++++ .../intuit/karate/core/ScenarioRuntime.java | 2 +- ...feature => ScenarioOutlineResultTest.java} | 0 .../core/parallel/parallel-outline-1.feature | 6 +++ .../src/test/java/demo/hooks/called.feature | 2 +- .../src/test/java/demo/hooks/hooks.feature | 13 +++++- 13 files changed, 149 insertions(+), 13 deletions(-) create mode 100644 karate-core/src/main/java/com/intuit/karate/core/AfterHookType.java rename karate-core/src/test/java/com/intuit/karate/core/{ScenarioOutlineResultTest.feature => ScenarioOutlineResultTest.java} (100%) diff --git a/README.md b/README.md index 4052be1ed..0438f5667 100755 --- a/README.md +++ b/README.md @@ -2252,6 +2252,7 @@ You can adjust configuration settings for the HTTP client used by Karate using t `printEnabled` | boolean | Can be used to suppress the [`print`](#print) output when not in 'dev mode' by setting as `false` (default `true`) `report` | JSON / boolean | see [report verbosity](#report-verbosity) `afterScenario` | JS function | Will be called [after every `Scenario`](#hooks) (or `Example` within a `Scenario Outline`), refer to this example: [`hooks.feature`](karate-demo/src/test/java/demo/hooks/hooks.feature) +`afterScenarioOutline` | JS function | Will be called [after every `Scenario Outline`](#hooks). Is called after the last `afterScenario` for the last scenario in the outline. Refer to this example: [`hooks.feature`](karate-demo/src/test/java/demo/hooks/hooks.feature) `afterFeature` | JS function | Will be called [after every `Feature`](#hooks), refer to this example: [`hooks.feature`](karate-demo/src/test/java/demo/hooks/hooks.feature) `ssl` | boolean | Enable HTTPS calls without needing to configure a trusted certificate or key-store. `ssl` | string | Like above, but force the SSL algorithm to one of [these values](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#SSLContext). (The above form internally defaults to `TLS` if simply set to `true`). @@ -4441,6 +4442,7 @@ Before *everything* (or 'globally' once) | See [`karate.callSingle()`](#karateca Before every `Scenario` | Use the [`Background`](#script-structure). Note that [`karate-config.js`](#karate-configjs) is processed before *every* `Scenario` - so you can choose to put "global" config here, for example using [`karate.configure()`](#karate-configure). Once (or at the start of) every `Feature` | Use a [`callonce`](#callonce) in the [`Background`](#script-structure). The advantage is that you can set up variables (using [`def`](#def) if needed) which can be used in all `Scenario`-s within that `Feature`. After every `Scenario` | [`configure afterScenario`](#configure) (see [example](karate-demo/src/test/java/demo/hooks/hooks.feature)) +After every `Scenario Outline` | [`configure afterScenarioOutline`](#configure) (see [example](karate-demo/src/test/java/demo/hooks/hooks.feature)) At the end of the `Feature` | [`configure afterFeature`](#configure) (see [example](karate-demo/src/test/java/demo/hooks/hooks.feature)) > Note that for the `afterFeature` hook to work, you should be using the [`Runner` API](#parallel-execution) and not the JUnit runner. diff --git a/karate-core/src/main/java/com/intuit/karate/RuntimeHook.java b/karate-core/src/main/java/com/intuit/karate/RuntimeHook.java index a0c648d02..83f26a7cb 100644 --- a/karate-core/src/main/java/com/intuit/karate/RuntimeHook.java +++ b/karate-core/src/main/java/com/intuit/karate/RuntimeHook.java @@ -46,6 +46,10 @@ default void afterScenario(ScenarioRuntime sr) { } + default void afterScenarioOutline(ScenarioRuntime sr) { + + } + default boolean beforeFeature(FeatureRuntime fr) { return true; } diff --git a/karate-core/src/main/java/com/intuit/karate/core/AfterHookType.java b/karate-core/src/main/java/com/intuit/karate/core/AfterHookType.java new file mode 100644 index 000000000..3febf64a8 --- /dev/null +++ b/karate-core/src/main/java/com/intuit/karate/core/AfterHookType.java @@ -0,0 +1,45 @@ +/* + * The MIT License + * + * Copyright 2022 Karate Labs Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.core; + +/** + * + * @author OwenK2 + */ +public enum AfterHookType { + + AFTER_SCENARIO("afterScenario"), + AFTER_OUTLINE("afterScenarioOutline"), + AFTER_FEATURE("afterFeature"); + + private String prefix; + + private AfterHookType(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } +} \ No newline at end of file diff --git a/karate-core/src/main/java/com/intuit/karate/core/Config.java b/karate-core/src/main/java/com/intuit/karate/core/Config.java index 9dbd7488f..3925933dd 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/Config.java +++ b/karate-core/src/main/java/com/intuit/karate/core/Config.java @@ -93,6 +93,7 @@ public class Config { private HttpLogModifier logModifier; private Variable afterScenario = Variable.NULL; + private Variable afterScenarioOutline = Variable.NULL; private Variable afterFeature = Variable.NULL; private Variable headers = Variable.NULL; private Variable cookies = Variable.NULL; @@ -175,6 +176,9 @@ public boolean configure(String key, Variable value) { // TODO use enum case "afterScenario": afterScenario = value; return false; + case "afterScenarioOutline": + afterScenarioOutline = value; + return false; case "afterFeature": afterFeature = value; return false; @@ -382,6 +386,7 @@ public Config(Config parent) { cookies = parent.cookies; responseHeaders = parent.responseHeaders; afterScenario = parent.afterScenario; + afterScenarioOutline = parent.afterScenarioOutline; afterFeature = parent.afterFeature; continueOnStepFailureMethods = parent.continueOnStepFailureMethods; continueAfterContinueOnStepFailure = parent.continueAfterContinueOnStepFailure; @@ -538,6 +543,14 @@ public void setAfterScenario(Variable afterScenario) { this.afterScenario = afterScenario; } + public Variable getAfterScenarioOutline() { + return afterScenarioOutline; + } + + public void setAfterScenarioOutline(Variable afterScenarioOutline) { + this.afterScenarioOutline = afterScenarioOutline; + } + public Variable getAfterFeature() { return afterFeature; } diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java index d4a5d7638..cf8387c4b 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java @@ -195,16 +195,38 @@ private void processScenario(ScenarioRuntime sr) { if (!sr.result.getStepResults().isEmpty()) { synchronized (result) { result.addResult(sr.result); + + // Execute afterScenarioOutline if applicable + // NOTE: Needs to be run after adding result, since result count is used to deterime + // if the scenario is the last in the outline + if (!sr.dryRun && isLastScenarioInOutline(sr.scenario)) { + sr.engine.invokeAfterHookIfConfigured(AfterHookType.AFTER_OUTLINE); + suite.hooks.forEach(h -> h.afterScenarioOutline(sr)); + } } } } } + private boolean isLastScenarioInOutline(Scenario scenario) { + // Check if scenario is part of an outline + if (!scenario.isOutlineExample()) return false; + + // Count the number of completed scenarios with the same section ID (in same outline) + int completedScenarios = 0; + for (ScenarioResult result : result.getScenarioResults()) { + if (result.getScenario().getSection().getIndex() == scenario.getSection().getIndex()) { + completedScenarios++; + } + } + return completedScenarios == scenario.getSection().getScenarioOutline().getNumScenarios(); + } + // extracted for junit5 public synchronized void afterFeature() { result.sortScenarioResults(); if (lastExecutedScenario != null) { - lastExecutedScenario.engine.invokeAfterHookIfConfigured(true); + lastExecutedScenario.engine.invokeAfterHookIfConfigured(AfterHookType.AFTER_FEATURE); result.setVariables(lastExecutedScenario.engine.getAllVariablesAsMap()); result.setConfig(lastExecutedScenario.engine.getConfig()); } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioEngine.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioEngine.java index 24c18be84..ebd9c22f0 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioEngine.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioEngine.java @@ -235,20 +235,35 @@ public void print(String exp) { evalJs("karate.log('[print]'," + exp + ")"); } - public void invokeAfterHookIfConfigured(boolean afterFeature) { + public void invokeAfterHookIfConfigured(AfterHookType hookType) { + // Do not call hooks on "called" scenarios/features if (runtime.caller.depth > 0) { return; } - Variable v = afterFeature ? config.getAfterFeature() : config.getAfterScenario(); + + // Get hook variable based on type + Variable v; + switch (hookType) { + case AFTER_SCENARIO: + v = config.getAfterScenario(); + break; + case AFTER_OUTLINE: + v = config.getAfterScenarioOutline(); + break; + case AFTER_FEATURE: + v = config.getAfterFeature(); + break; + default: return; + } + if (v.isJsOrJavaFunction()) { - if (afterFeature) { + if (hookType == AfterHookType.AFTER_FEATURE) { ScenarioEngine.set(this); // for any bridge / js to work } try { executeFunction(v); } catch (Exception e) { - String prefix = afterFeature ? "afterFeature" : "afterScenario"; - logger.warn("{} hook failed: {}", prefix, e + ""); + logger.warn("{} hook failed: {}", hookType.getPrefix(), e + ""); } } } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java index c8b16c8f1..50833b9fe 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioOutlineResult.java @@ -58,14 +58,18 @@ public Map toKarateJson() { List> scenarioResults = new ArrayList(); if (runtime.featureRuntime != null && runtime.featureRuntime.result != null) { // Add all past results - runtime.featureRuntime.result.getScenarioResults().forEach(result -> { + boolean needToAddRecent = runtime.result != null; + for(ScenarioResult result : runtime.featureRuntime.result.getScenarioResults()) { if (result.getScenario().getSection().getIndex() == scenarioOutline.getSection().getIndex()) { scenarioResults.add(result.toInfoJson()); + if(result.equals(runtime.result)) { + needToAddRecent = false; + } } - }); + } - // Add most recent result - if (runtime.result != null) { + // Add most recent result if we haven't already (and it's not null) + if (needToAddRecent) { scenarioResults.add(runtime.result.toInfoJson()); } } diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java index 1786374b5..1a85043c4 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java @@ -61,6 +61,20 @@ public int compareTo(ScenarioResult sr) { return scenario.getExampleIndex() - sr.scenario.getExampleIndex(); } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + return compareTo((ScenarioResult)obj) == 0; + } + public String getFailureMessageForDisplay() { if (failedStep == null) { return null; diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java index 7c290d8ed..096a6f330 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java @@ -505,7 +505,7 @@ public void afterRun() { currentStepResult = result.addFakeStepResult("no steps executed", null); } if (!dryRun) { - engine.invokeAfterHookIfConfigured(false); + engine.invokeAfterHookIfConfigured(AfterHookType.AFTER_SCENARIO); featureRuntime.suite.hooks.forEach(h -> h.afterScenario(this)); engine.stop(currentStepResult); } diff --git a/karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.feature b/karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.java similarity index 100% rename from karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.feature rename to karate-core/src/test/java/com/intuit/karate/core/ScenarioOutlineResultTest.java diff --git a/karate-core/src/test/java/com/intuit/karate/core/parallel/parallel-outline-1.feature b/karate-core/src/test/java/com/intuit/karate/core/parallel/parallel-outline-1.feature index 225ec6165..c1ed42389 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/parallel/parallel-outline-1.feature +++ b/karate-core/src/test/java/com/intuit/karate/core/parallel/parallel-outline-1.feature @@ -13,6 +13,12 @@ Background: function fn() { console.log('afterFeature'); } + """ + * configure afterScenarioOutline = + """ + function fn() { + console.log('afterScenarioOutline'); + } """ * configure afterScenario = """ diff --git a/karate-demo/src/test/java/demo/hooks/called.feature b/karate-demo/src/test/java/demo/hooks/called.feature index aabbbe373..7c67ac900 100644 --- a/karate-demo/src/test/java/demo/hooks/called.feature +++ b/karate-demo/src/test/java/demo/hooks/called.feature @@ -2,7 +2,7 @@ Feature: Background: -# 'afterScenario' and 'afterFeature' are NOT supported when a feature is called +# 'afterScenario', 'afterScenarioOutline', and 'afterFeature' are NOT supported when a feature is called # so this will have no effect, UNLESS this feature is run directly * configure afterScenario = function(){ karate.log('end called scenario') } diff --git a/karate-demo/src/test/java/demo/hooks/hooks.feature b/karate-demo/src/test/java/demo/hooks/hooks.feature index 70d847e4f..12e01269a 100644 --- a/karate-demo/src/test/java/demo/hooks/hooks.feature +++ b/karate-demo/src/test/java/demo/hooks/hooks.feature @@ -1,5 +1,5 @@ Feature: demo karate's equivalent of before and after hooks - note that 'afterScenario' / 'afterFeature' if set up using 'configure' + note that 'afterScenario' / 'afterScenarioOutline' / 'afterFeature' if set up using 'configure' is not supported within features invoked using the 'call' or 'callonce' keywords Background: @@ -28,6 +28,17 @@ function(){ * configure afterFeature = function(){ karate.call('after-feature.feature'); } +# Only runs at the end of a scenario outline after all examples have been run +# This hook will be called after the last scenario in the outline is executed +# It will also run after any configured 'afterScenario's for that outline +# NOTE if using parallel, last Scenario executed may not be the last example in the outline +* configure afterScenarioOutline = +""" +function(){ + karate.log('after scenario outline:', karate.scenarioOutline.name); +} +""" + Scenario: first * print foo