Merge branch '1.2.17-SNAPSHOT'

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.secdec.astam-correlator</groupId>
     <artifactId>master-pom</artifactId>
-    <version>1.2.16</version>
+    <version>1.2.17</version>
     <packaging>pom</packaging>
 
     <name>ThreadFix</name>
@@ -37,7 +37,7 @@
     <licenses>
         <license>
             <name>Mozilla Public License v2.0</name>
-            <url>https://www.mozilla.org/un-US/MPL/2.0/</url>
+            <url>https://www.mozilla.org/en-US/MPL/2.0/</url>
         </license>
     </licenses>
 
@@ -426,7 +426,7 @@
         <maven-failsafe.version>2.17</maven-failsafe.version>
         <surefire-junit47.version>2.17</surefire-junit47.version>
         <jacoco.version>0.7.0.201403182114</jacoco.version>
-        <jsoup.version>1.8.2</jsoup.version>
+        <jsoup.version>1.8.3</jsoup.version>
         <gson.version>2.2.4</gson.version>
 
         <!-- Testing Properties -->

ssvl-converter/pom.xml

@@ -7,11 +7,11 @@
     <parent>
         <groupId>com.github.secdec.astam-correlator</groupId>
         <artifactId>master-pom</artifactId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
 
     <artifactId>ssvl-converter</artifactId>
-    <version>1.2.16</version>
+    <version>1.2.17</version>
 
     <build>
         <sourceDirectory>src/main/java</sourceDirectory>

threadfix-astam/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>com.github.secdec.astam-correlator</groupId>
         <artifactId>master-pom</artifactId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

threadfix-cli-importers/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>master-pom</artifactId>
         <groupId>com.github.secdec.astam-correlator</groupId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

threadfix-cli-importers/src/test/java/com/denimgroup/threadfix/framework/dotNetWebForm/EndToEndTests.java

@@ -104,32 +104,33 @@ public void assertDynamicXSSFindsEndpoint() {
         assert foundBasicEndpoint : "Didn't find /WebForm1.aspx";
     }
 
-    @Test
-    public void testXSSVulnsMerge() {
-        Application application = getApplication(TestConstants.WEB_FORMS_DROP_DOWN,
-                ScanLocationManager.getRoot(),
-                "SBIR/webform.xml", "SBIR/webform.fpr");
-
-        List<Scan> scans = application.getScans();
-        assert scans.size() == 2 :
-                "Got " + scans.size() + " scans instead of 2.";
-
-        boolean hasMergedXSSVuln = false;
-
-        for (Vulnerability vulnerability : application.getVulnerabilities()) {
-            if (vulnerability.getGenericVulnerability().getDisplayId().equals(79)) {
-                if (vulnerability.getFindings().size() == 2) {
-                    hasMergedXSSVuln = true;
-                    System.out.println("Found it!");
-                } else {
-                    System.out.println("Found a XSS vuln but it didn't have 2 findings. " +
-                            "It had " + vulnerability.getFindings().size());
-                }
-            }
-        }
-
-        assert hasMergedXSSVuln : "Didn't find a merged vulnerability.";
-    }
+    //  This test project was not provided with the public Threadfix release and has not been found online.
+//    @Test
+//    public void testXSSVulnsMerge() {
+//        Application application = getApplication(TestConstants.WEB_FORMS_DROP_DOWN,
+//                ScanLocationManager.getRoot(),
+//                "SBIR/webform.xml", "SBIR/webform.fpr");
+//
+//        List<Scan> scans = application.getScans();
+//        assert scans.size() == 2 :
+//                "Got " + scans.size() + " scans instead of 2.";
+//
+//        boolean hasMergedXSSVuln = false;
+//
+//        for (Vulnerability vulnerability : application.getVulnerabilities()) {
+//            if (vulnerability.getGenericVulnerability().getDisplayId().equals(79)) {
+//                if (vulnerability.getFindings().size() == 2) {
+//                    hasMergedXSSVuln = true;
+//                    System.out.println("Found it!");
+//                } else {
+//                    System.out.println("Found a XSS vuln but it didn't have 2 findings. " +
+//                            "It had " + vulnerability.getFindings().size());
+//                }
+//            }
+//        }
+//
+//        assert hasMergedXSSVuln : "Didn't find a merged vulnerability.";
+//    }
 
     private Application getApplication(String sourceLocation, String scanBase, String... scans) {
 

threadfix-cli-lib/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.github.secdec.astam-correlator</groupId>
         <artifactId>master-pom</artifactId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
 
     <artifactId>threadfix-cli-lib</artifactId>

threadfix-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.github.secdec.astam-correlator</groupId>
         <artifactId>master-pom</artifactId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
 
     <artifactId>threadfix-cli</artifactId>

threadfix-data-access/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>master-pom</artifactId>
         <groupId>com.github.secdec.astam-correlator</groupId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

threadfix-data-migration/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>master-pom</artifactId>
         <groupId>com.github.secdec.astam-correlator</groupId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

threadfix-entities/pom.xml

@@ -97,7 +97,7 @@
     <parent>
         <groupId>com.github.secdec.astam-correlator</groupId>
         <artifactId>master-pom</artifactId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
 
     <dependencies>

threadfix-entities/src/main/java/com/denimgroup/threadfix/data/entities/WildcardEndpointPathNode.java

@@ -7,12 +7,15 @@
 
 public class WildcardEndpointPathNode implements EndpointPathNode {
 
+    private static Pattern namedGroupPattern = Pattern.compile("\\(\\?P\\<\\w+\\>");
+
     private Pattern wildcardPattern;
 
     public WildcardEndpointPathNode(String pattern) {
     	if (pattern == null) {
     		this.wildcardPattern = Pattern.compile(".*");
 	    } else {
+    	    pattern = namedGroupPattern.matcher(pattern).replaceAll("(");
 		    this.wildcardPattern = Pattern.compile(pattern);
 	    }
     }

threadfix-ham/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.github.secdec.astam-correlator</groupId>
         <artifactId>master-pom</artifactId>
-        <version>1.2.16</version>
+        <version>1.2.17</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -188,7 +188,7 @@
         <dependency>
             <groupId>org.jsoup</groupId>
             <artifactId>jsoup</artifactId>
-            <version>1.8.3</version>
+            <version>${jsoup.version}</version>
         </dependency>
     </dependencies>
 

threadfix-ham/src/main/java/com/denimgroup/threadfix/framework/engine/ProjectDirectory.java

@@ -300,7 +300,11 @@ public String findCanonicalFilePath(@Nonnull File file) {
             filePath = file.getAbsolutePath().substring(directory.getAbsolutePath().length());
         }
 
-        return filePath;
+        if (filePath != null) {
+            return FilePathUtils.normalizePath(filePath);
+        } else {
+            return null;
+        }
     }
 
     @Nullable
@@ -334,7 +338,11 @@ private String calculateBestOption(@Nonnull String[] pathSegments, @Nonnull Set<
             }
         }
 
-        return returnOption;
+        if (returnOption != null) {
+            return FilePathUtils.normalizePath(returnOption);
+        } else {
+            return null;
+        }
     }
 
     // split along / or \ or just return the whole path

threadfix-ham/src/main/java/com/denimgroup/threadfix/framework/engine/framework/FrameworkCalculator.java

@@ -51,8 +51,8 @@ public class FrameworkCalculator {
     static {
         // TODO detect language first and use that to narrow down the frameworks
         register(new JavaAndJspFrameworkChecker());
-        register(new WebFormsFrameworkChecker());
         register(new DotNetFrameworkChecker());
+        register(new WebFormsFrameworkChecker());
         register(new RailsFrameworkChecker());
         register(new DjangoFrameworkChecker());
     }

threadfix-ham/src/main/java/com/denimgroup/threadfix/framework/engine/framework/JavaAndJspFrameworkChecker.java

@@ -90,15 +90,6 @@ private boolean checkJsp(@Nonnull ProjectDirectory directory) {
     @SuppressWarnings("unchecked")
     public FrameworkType check(@Nonnull ProjectDirectory directory) {
 
-        FrameworkType frameworkType = checkMappings(directory);
-        if (frameworkType != FrameworkType.NONE) {
-            return frameworkType;
-        }
-
-        if (checkSpringMvc(directory)) {
-            return FrameworkType.SPRING_MVC;
-        }
-
         if (checkStruts(directory)) {
             return FrameworkType.STRUTS;
         }
@@ -114,6 +105,11 @@ public FrameworkType check(@Nonnull ProjectDirectory directory) {
             return FrameworkType.JSP;
         }
 
+        FrameworkType frameworkType = checkMappings(directory);
+        if (frameworkType != FrameworkType.NONE) {
+            return frameworkType;
+        }
+
         return FrameworkType.NONE;
     }
 

threadfix-ham/src/main/java/com/denimgroup/threadfix/framework/engine/full/GeneratorBasedEndpointDatabase.java

@@ -25,11 +25,13 @@
 ////////////////////////////////////////////////////////////////////////
 package com.denimgroup.threadfix.framework.engine.full;
 
+import com.denimgroup.threadfix.data.enums.EndpointRelevanceStrictness;
 import com.denimgroup.threadfix.data.enums.FrameworkType;
 import com.denimgroup.threadfix.data.enums.InformationSourceType;
 import com.denimgroup.threadfix.data.interfaces.Endpoint;
 import com.denimgroup.threadfix.framework.engine.CodePoint;
 import com.denimgroup.threadfix.framework.engine.cleaner.PathCleaner;
+import com.denimgroup.threadfix.framework.util.EndpointUtil;
 import com.denimgroup.threadfix.logging.SanitizedLogger;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -43,6 +45,9 @@ class GeneratorBasedEndpointDatabase implements EndpointDatabase {
     @Nonnull
     private final List<Endpoint> endpoints;
 
+    @Nonnull
+    private final List<Endpoint> flattenedEndpoints;
+
     @Nonnull
     private final PathCleaner pathCleaner;
 
@@ -64,6 +69,7 @@ public GeneratorBasedEndpointDatabase(@Nonnull EndpointGenerator endpointGenerat
         log.info("Using generic EndpointGenerator-based translator.");
 
         endpoints = endpointGenerator.generateEndpoints();
+        flattenedEndpoints = EndpointUtil.flattenWithVariants(endpoints);
 
         log.info("Found ".concat(String.valueOf(endpoints.size())).concat(" endpoints:"));
         for (int i = 0; i < endpoints.size(); i++) {
@@ -83,7 +89,7 @@ public GeneratorBasedEndpointDatabase(@Nonnull EndpointGenerator endpointGenerat
 
     private void buildMappings() {
         log.info("Building mappings.");
-        for (Endpoint endpoint : endpoints) {
+        for (Endpoint endpoint : flattenedEndpoints) {
             addToMap(dynamicMap, endpoint.getUrlPath(), endpoint);
             addToMap(staticMap, endpoint.getFilePath(), endpoint);
 
@@ -121,19 +127,20 @@ public Endpoint findBestMatch(@Nonnull EndpointQuery query) {
         Endpoint bestEndpoint = null;
         int bestEndpointRelevance = -1;
 
-        if (query.getDynamicPath() == null) {
-            return null;
-        }
-        String dynamicPath = pathCleaner.cleanDynamicPath(query.getDynamicPath());
-
         Set<Endpoint> endpoints = findAllMatches(query);
 
-        for (Endpoint currentEndpoint : endpoints) {
-            int relevance = currentEndpoint.compareRelevance(dynamicPath);
-            if (relevance > bestEndpointRelevance) {
-                bestEndpoint = currentEndpoint;
-                bestEndpointRelevance = relevance;
+        if (query.getDynamicPath() != null) {
+            String dynamicPath = pathCleaner.cleanDynamicPath(query.getDynamicPath());
+            for (Endpoint currentEndpoint : endpoints) {
+                int relevance = currentEndpoint.compareRelevance(dynamicPath);
+                if (relevance > bestEndpointRelevance && currentEndpoint.isRelevant(dynamicPath, EndpointRelevanceStrictness.STRICT)) {
+                    bestEndpoint = currentEndpoint;
+                    bestEndpointRelevance = relevance;
+                }
             }
+        } else if (!endpoints.isEmpty()) {
+            bestEndpoint = endpoints.iterator().next();
+            bestEndpointRelevance = 1;
         }
 
         if (bestEndpointRelevance > 0) {
@@ -166,6 +173,7 @@ public Set<Endpoint> findAllMatches(@Nonnull EndpointQuery query) {
         if (!useStatic && query.getDynamicPath() != null) {
             String cleaned = pathCleaner.cleanDynamicPath(query.getDynamicPath());
             resultSets.add(getValueOrEmptySet(cleaned, dynamicMap));
+            resultSets.addAll(list(findEligibleEndpoints(cleaned)));
         }
 
         if (useStatic && query.getStaticPath() != null) {
@@ -184,7 +192,7 @@ public Set<Endpoint> findAllMatches(@Nonnull EndpointQuery query) {
 
         if (resultSets.size() > 0) {
             for (Set<Endpoint> endpoints : resultSets) {
-                if (endpoints != null) {
+                if (endpoints != null && !endpoints.isEmpty()) {
 
                     if (!assignedInitial) {
                         resultingSet = endpoints;
@@ -221,8 +229,6 @@ public Set<Endpoint> findAllMatches(@Nonnull EndpointQuery query) {
             resultingSet.addAll(fromCodePoints);
         }
 
-        resultingSet.addAll(findEligibleEndpoints(pathCleaner.cleanDynamicPath(query.getDynamicPath())));
-
         return resultingSet;
     }
 
@@ -297,8 +303,8 @@ private Set<Endpoint> getValueOrEmptySetWithSimpleKey(@Nullable String key,
 
     private Set<Endpoint> findEligibleEndpoints(String endpointPath) {
         Set<Endpoint> result = set();
-        for (Endpoint endpoint : endpoints) {
-            if (endpoint.compareRelevance(endpointPath) > 0) {
+        for (Endpoint endpoint : flattenedEndpoints) {
+            if (endpoint.isRelevant(endpointPath, EndpointRelevanceStrictness.STRICT)) {
                 result.add(endpoint);
             }
         }

threadfix-ham/src/main/java/com/denimgroup/threadfix/framework/impl/django/DjangoEndpoint.java

@@ -287,4 +287,17 @@ public boolean matchesLineNumber(int lineNumber) {
     protected List<String> getLintLine() {
         return null;
     }
+
+    @Nonnull
+    @Override
+    public String toString() {
+        return
+            urlPath +
+            " - " +
+            (filePath == null ? "<no source file>" : filePath) +
+            ":" +
+            startLineNumber +
+            "," +
+            endLineNumber;
+    }
 }