add flow support in template (i.e javascript scripting) (#4015)

* add flow logic

* progress

* working POC

* fix string slice normalization issue in variables

* update

* fix nil panic

* remove poll()

* load file with sandbox and more

* fix failing integration tests

* JS: log: print in vardump format

* fix missing id in protocols

* fix proto prefix in template context

* flow: add unit tests

* conditional flow support using flow

* fix proto callbacks + more unit tests

* adds integration test

* conditional flow: check if req has any matchers

* fix lint error

* deprecate iterate-all+ missing multi-proto implementation

* fix ip input in raw request

* JS: feat dedupe object+ more builtin funcs

* feat: hide protocol result using hide

* feat: async execution

* complete async execution support

* fix condition-flow without any matchers

* refactor: template executer package (tmplexec)

* flow executor working

* fix data race in templateCtx

* templateCtx redesign

* fix failing unit test

* add multiprotocol support to deprecated syntax

* fix race condition in utils & tlsx

* add documentation in flow package

* remove regions.txt file

* fix minor issue with self contained templates

* fix typos of copilot

* dep + misc update

* fix reqID: use req.Type instead of template.Type

---------

Co-authored-by: sandeep <[email protected]>

integration_tests/flow/conditional-flow-negative.yaml

@@ -0,0 +1,27 @@
+id: ghost-blog-detection
+info:
+  name: Ghost blog detection
+  author: pdteam
+  severity: info
+
+
+flow: dns() && http()
+
+dns:
+  - name: "{{FQDN}}"
+    type: CNAME
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"

integration_tests/flow/conditional-flow.yaml

@@ -0,0 +1,27 @@
+id: ghost-blog-detection
+info:
+  name: Ghost blog detection
+  author: pdteam
+  severity: info
+
+
+flow: dns() && http()
+
+dns:
+  - name: "{{FQDN}}"
+    type: CNAME
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "ghost.io"

integration_tests/flow/dns-ns-probe.yaml

@@ -0,0 +1,42 @@
+id: dns-ns-probe
+
+info:
+  name: Nuclei flow dns ns probe
+  author: pdteam
+  severity: info
+  description: Description of the Template
+  reference: https://example-reference-link
+
+flow: |
+  dns("fetch-ns");
+  for(let ns of template["nameservers"]) {
+    set("nameserver",ns);
+    dns("probe-ns");
+  };
+
+dns:
+  - id: "fetch-ns"
+    name: "{{FQDN}}"
+    type: NS
+    matchers:
+      - type: word
+        words:
+          - "IN\tNS"
+    extractors:
+      - type: regex
+        internal: true
+        name: "nameservers"
+        group: 1
+        regex:
+          - "IN\tNS\t(.+)"
+
+  - id: "probe-ns"
+    name: "{{nameserver}}"
+    type: A
+    class: inet
+    retries: 3
+    recursion: true
+    extractors:
+      - type: dsl
+        dsl:
+          - "a"

integration_tests/flow/iterate-values-flow.yaml

@@ -0,0 +1,35 @@
+id: extract-emails
+
+info:
+  name: Extract Email IDs from Response
+  author: pdteam
+  severity: info
+
+
+flow: |
+  http(0)
+  for(let email of template["emails"]) {
+    set("email",email);
+    http(1);
+  }
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    extractors:
+      - type: regex
+        name: emails
+        internal: true
+        regex:
+          - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
+
+  - method: GET
+    path:
+      - "{{BaseURL}}/user/{{base64(email)}}"
+
+    matchers:
+      - type: word
+        words:
+          - "Welcome"

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4b0a00483046022100cd2b9d34169cdb716caee25976fed763880435f2f1e2979c9d7c9d2bd7b8e409022100dd0ba8bd3fa6a6be5f964ca3b0ce8bdbb20d865553133cf494ef64fbeebff345
+# digest: 4a0a00473045022100e17c7a809fd64419baf401b5331edab3a68a4c182f7777614beb1862eb6ea8b7022011b95fc0e22d7f82e08e01b56ce87afdbe03027c238ba290a058d695226173ae

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a00483046022100f663e5afaf5c118b21b9c5918cba12d7cc83edc2a3ee0f338c07e3cd1fe40e20022100b46193e3275c490a4ad3897c6e2ca51ce09f408538b17d041e0063d40f4df833
+# digest: 490a004630440220241d7faae14ab5760dbe7acf621a3923d0650148bc14a52a9be06ba76e8e0abf02201885fcc432d354d3c99ea97b964838719451bc97f148229f187f61eee7525eb6

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4b0a00483046022100c45cd27b9d49879663e1ea3c877dc362d06b8a0aea64b1ab06be3af5aa9a32ee0221008f5ee347245a2c1e04c46528e4c70a5a851f95c6ba49d2834ef7c3784bca47a9
+# digest: 490a004630440220427cb7100f0b7d95224f490a6f4f626186782cb26c69f2551d6aefcdbc7c17d20220206161ad3a98afe8fcef9dd06d9a6dd5f34c5f7e3cd3ab7f81328f033dcd2b48

integration_tests/protocols/code/py-snippet.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a00473045022100df57bf446d6d8e73ff9424b1055faebcea9038e5d5934834ed8e619b77bdfd5e02201754c1cebe9f65883315b3830755a0689999f33db7102cd8d5469e4c01cc6a66
+# digest: 4a0a00473045022056092462597e85139626656d37df123094cb3861bdf583450c38814bac8df9cb022100e83a8c552f8f8a098f6b7ec8a32c6b448b995e000884beadb50cb0f2720117de

integration_tests/protocols/http/matcher-status.yaml

@@ -1,4 +1,4 @@
-id: matchet-status
+id: matcher-status
 
 info:
   name: Test Matcher Status

v2/cmd/integration-test/flow.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+)
+
+var flowTestcases = []TestCaseInfo{
+	{Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}},
+	{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}},
+	{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}},
+	{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}},
+}
+
+type conditionalFlow struct{}
+
+func (t *conditionalFlow) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 2)
+}
+
+type conditionalFlowNegative struct{}
+
+func (t *conditionalFlowNegative) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}
+
+type iterateValuesFlow struct{}
+
+func (t *iterateValuesFlow) Execute(filePath string) error {
+	router := httprouter.New()
+	testemails := []string{
+		"[email protected]",
+		"[email protected]",
+	}
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(fmt.Sprint(testemails)))
+	})
+	router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
+	})
+
+	router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
+	})
+
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 2)
+}
+
+type dnsNsProbe struct{}
+
+func (t *dnsNsProbe) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 3)
+}
+
+func getBase64(input string) string {
+	return base64.StdEncoding.EncodeToString([]byte(input))
+}

v2/cmd/integration-test/integration-test.go

@@ -49,6 +49,7 @@ var (
 		"multi":           multiProtoTestcases,
 		"generic":         genericTestcases,
 		"dsl":             dslTestcases,
+		"flow":            flowTestcases,
 	}
 
 	// For debug purposes

v2/cmd/integration-test/multi.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+)
 
 var multiProtoTestcases = []TestCaseInfo{
 	{Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}},

v2/cmd/integration-test/template-path.go

@@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{
 	//template folder path issue
 	{Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}},
 	//cwd
-	{Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
+	{Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
 	//relative path
 	{Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}},
 	//absolute path