DHFPROD-4510: Fixing failHard param

Also removed some dead code from the mapping step; noticed that when I added an error to the step during manual testing. 

Slight mod to root .gitignore file - I'm trying out VSCode for Java coding, and it defaults to compiling classes to bin directory.

.gitignore

@@ -7,7 +7,7 @@
 npm-debug.log
 marklogic-data-hub/bin/
 marklogic-data-hub/src/main/resources/ml-modules/root/trace-ui
-web/bin/
+bin
 ml-data-hub-plugin/bin
 build/
 out/
@@ -29,7 +29,6 @@ node_modules/
 *.iws
 *.db
 ye-olde-project
-/bin/
 examples/dh-5-example/input
 installer.log
 .java-version

marklogic-data-hub/src/main/java/com/marklogic/hub/cli/client/CommandLineFlowInputs.java

@@ -139,7 +139,7 @@ protected Map<String, Object> buildStepConfig(StringBuilder runFlowString) {
     }
 
     protected Map<String, Object> buildFlowOptions() {
-        String optionsString = null;
+        String optionsString;
         if (StringUtils.isNotEmpty(optionsFile)) {
             try {
                 optionsString = new String(FileCopyUtils.copyToByteArray(new File(optionsFile)));
@@ -150,15 +150,24 @@ protected Map<String, Object> buildFlowOptions() {
             optionsString = optionsJSON;
         }
 
+        Map<String, Object> optionsMap = null;
         if (StringUtils.isNotEmpty(optionsString)) {
             try {
-                return new ObjectMapper().readValue(optionsString, new TypeReference<Map<String, Object>>() {
-                });
+                optionsMap = new ObjectMapper().readValue(optionsString, new TypeReference<Map<String, Object>>() {});
             } catch (IOException ex) {
                 throw new RuntimeException("Unable to parse JSON options string: " + optionsString, ex);
             }
         }
-        return null;
+
+        // Needed to force the flow to stop, not just the step
+        if (this.failHard) {
+            if (optionsMap == null) {
+                optionsMap = new HashMap<>();
+            }
+            optionsMap.put("stopOnError", true);
+        }
+
+        return optionsMap;
     }
 
     public String getFlowName() {

marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java

@@ -184,6 +184,8 @@ protected RunFlowResponse runFlow(Flow flow, List<String> stepNums, String jobId
             disableJobOutput = false;
         }
 
+        configureStopOnError(flow, options);
+
         if(stepNums == null) {
             stepNums = new ArrayList<>(flow.getSteps().keySet());
         }
@@ -228,6 +230,22 @@ protected RunFlowResponse runFlow(Flow flow, List<String> stepNums, String jobId
         return response;
     }
 
+    /**
+     * To support the "failHard" parameter in CommandLineFlowInputs, this method checks for stopOnError in the options
+     * map. If true, this will set stopOnError in the flow. That will cause no other steps to be run once a step fails.
+     *
+     * @param flow
+     * @param options
+     */
+    protected void configureStopOnError(Flow flow, Map<String, Object> options) {
+        if (options != null) {
+            Object value = options.get("stopOnError");
+            if (Boolean.TRUE.equals(value) || "true".equals(value)) {
+                flow.setStopOnError(true);
+            }
+        }
+    }
+
     private void initializeFlow(StepRunnerFactory stepRunnerFactory, String jobId) {
         //Reset the states to initial values before starting a flow run
         isRunning.set(true);

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/builtins/steps/mapping/entity-services/main.sjs

@@ -8,11 +8,8 @@ const xqueryLib = require('xquery-lib.xqy')
 
 // caching mappings in key to object since tests can have multiple mappings run in same transaction
 var mappings = {};
-var entityModel = null;
 
 function main(content, options) {
-  //let's set our output format, so we know what we're exporting
-  let inputFormat = options.inputFormat ? options.inputFormat.toLowerCase() : datahub.flow.consts.DEFAULT_FORMAT;
   let outputFormat = options.outputFormat ? options.outputFormat.toLowerCase() : datahub.flow.consts.DEFAULT_FORMAT;
   if (outputFormat !== datahub.flow.consts.JSON && outputFormat !== datahub.flow.consts.XML) {
     datahub.debug.log({

marklogic-data-hub/src/test/java/com/marklogic/hub/cli/client/CommandLineFlowInputsTest.java

@@ -11,8 +11,7 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class CommandLineFlowInputsTest {
 
@@ -47,6 +46,9 @@ public void test() {
 
         Map<String, Object> options = flowInputs.getOptions();
         assertEquals("world", options.get("hello"));
+        assertEquals(true, options.get("stopOnError"), "When failHard=true, stopOnError needs to be set to true " +
+            "so that the flow will be stopped when a step has a failure (whereas stopOnFailure only stops the step). " +
+            "stopOnError is added to the options because it's not step-level config, and thus shouldn't be in stepConfig.");
 
         Map<String, Object> stepConfig = flowInputs.getStepConfig();
         assertEquals(Boolean.TRUE, stepConfig.get("stopOnFailure"));
@@ -84,4 +86,27 @@ void readOptionsFromFile() throws IOException {
         assertEquals("green", values.get(1));
         assertEquals("blue", values.get(2));
     }
+
+    @Test
+    void onlyFailHardIsSet() {
+        CommandLineFlowInputs inputs = new CommandLineFlowInputs();
+        inputs.setFailHard(true);
+
+        FlowInputs flowInputs = inputs.buildFlowInputs().getLeft();
+        assertEquals(true, flowInputs.getOptions().get("stopOnError"), "An options map should be built even though " +
+            "the user didn't provide any options, since stopOnError needs to be added to it");
+        assertEquals(true, flowInputs.getStepConfig().get("stopOnFailure"));
+    }
+
+    @Test
+    void failHardIsFalse() {
+        CommandLineFlowInputs inputs = new CommandLineFlowInputs();
+        inputs.setFailHard(false);
+
+        FlowInputs flowInputs = inputs.buildFlowInputs().getLeft();
+        assertNull(flowInputs.getOptions(), "Since failHard was not set to true, " +
+            "stopOnError should not be in the options, and the options should be null since no other options were provided");
+        assertFalse(flowInputs.getStepConfig().containsKey("stopOnFailure"), "Since failHard was not set to true, " +
+            "stopOnFailure should not be in the stepConfig");
+    }
 }

marklogic-data-hub/src/test/java/com/marklogic/hub/flow/impl/FlowRunnerImplTest.java

@@ -1,9 +1,14 @@
 package com.marklogic.hub.flow.impl;
 
+import com.marklogic.hub.flow.Flow;
+import com.marklogic.hub.flow.FlowInputs;
 import com.marklogic.hub.flow.RunFlowResponse;
 import org.junit.jupiter.api.Test;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
 
 public class FlowRunnerImplTest {
 
@@ -24,4 +29,34 @@ void copyJobDataToResponse() {
         assertEquals("3", response.getLastAttemptedStep());
         assertEquals("2", response.getLastCompletedStep());
     }
+
+    @Test
+    void configureStopOnError() {
+        FlowRunnerImpl flowRunner = new FlowRunnerImpl();
+        Flow flow = new FlowImpl();
+
+        Map<String, Object> options = new HashMap<>();
+        options.put("stopOnError", true);
+        flowRunner.configureStopOnError(flow, options);
+        assertTrue(flow.isStopOnError());
+
+        flow = new FlowImpl();
+        options.put("stopOnError", "true");
+        flowRunner.configureStopOnError(flow, options);
+        assertTrue(flow.isStopOnError());
+
+        flow = new FlowImpl();
+        options.put("stopOnError", "true");
+        flowRunner.configureStopOnError(flow, options);
+        assertTrue(flow.isStopOnError());
+
+        flow = new FlowImpl();
+        options.put("stopOnError", "false");
+        flowRunner.configureStopOnError(flow, options);
+        assertFalse(flow.isStopOnError());
+
+        flow = new FlowImpl();
+        flowRunner.configureStopOnError(flow, new HashMap<>());
+        assertFalse(flow.isStopOnError());
+    }
 }

marklogic-data-hub/src/test/java/com/marklogic/hub/flow/impl/FlowRunnerTest.java

@@ -26,9 +26,14 @@
 import com.marklogic.client.query.DeleteQueryDefinition;
 import com.marklogic.client.query.QueryManager;
 import com.marklogic.hub.AbstractHubCoreTest;
+import com.marklogic.hub.FlowManager;
 import com.marklogic.hub.HubConfig;
+import com.marklogic.hub.dataservices.ArtifactService;
+import com.marklogic.hub.dataservices.FlowService;
+import com.marklogic.hub.flow.Flow;
 import com.marklogic.hub.flow.FlowInputs;
 import com.marklogic.hub.flow.RunFlowResponse;
+import com.marklogic.hub.impl.FlowManagerImpl;
 import com.marklogic.hub.job.JobStatus;
 import com.marklogic.hub.step.RunStepResponse;
 import org.junit.jupiter.api.Assertions;
@@ -374,21 +379,36 @@ public void testDisableJobOutput(){
 
     @Test
     public void testRunFlowStopOnError(){
-        runAsDataHubOperator();
+        verifyFlowStopsOnError(new HashMap<>());
+    }
 
-        Map<String,Object> opts = new HashMap<>();
+    @Test
+    void testStopOnErrorViaOptions() {
+        // Verify that stopOnError in options works by first modifying the flow so it doesn't have stopOnError=true
+        FlowManager mgr = new FlowManagerImpl(getHubConfig());
+        Flow flow = mgr.getFlow("testFlow");
+        flow.setStopOnError(false);
+        mgr.saveFlow(flow);
 
-        opts.put("targetDatabase", HubConfig.DEFAULT_STAGING_NAME);
-        opts.put("sourceDatabase", HubConfig.DEFAULT_STAGING_NAME);
+        Map<String, Object> options = new HashMap<>();
+        options.put("stopOnError", true);
+        verifyFlowStopsOnError(options);
+    }
+
+    private void verifyFlowStopsOnError(Map<String, Object> options) {
+        runAsDataHubOperator();
+
+        options.put("targetDatabase", HubConfig.DEFAULT_STAGING_NAME);
+        options.put("sourceDatabase", HubConfig.DEFAULT_STAGING_NAME);
         Map<String,String> mapping = new HashMap<>();
         mapping.put("name", "non-existent-mapping");
         mapping.put("version", "1");
-        opts.put("mapping", mapping);
+        options.put("mapping", mapping);
 
-        RunFlowResponse resp = runFlow("testFlow", "1,6", UUID.randomUUID().toString(), opts, null);
+        RunFlowResponse resp = runFlow("testFlow", "1,6", UUID.randomUUID().toString(), options, null);
         flowRunner.awaitCompletion();
-        Assertions.assertTrue(getDocCount(HubConfig.DEFAULT_STAGING_NAME, "xml-coll") == 1);
-        Assertions.assertTrue(JobStatus.STOP_ON_ERROR.toString().equalsIgnoreCase(resp.getJobStatus()));
+        assertEquals(1, getDocCount(HubConfig.DEFAULT_STAGING_NAME, "xml-coll"));
+        assertEquals(JobStatus.STOP_ON_ERROR.toString(), resp.getJobStatus());
     }
 
     @Test