-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add
flow
support in template (i.e javascript scripting) (#4015)
* add flow logic * progress * working POC * fix string slice normalization issue in variables * update * fix nil panic * remove poll() * load file with sandbox and more * fix failing integration tests * JS: log: print in vardump format * fix missing id in protocols * fix proto prefix in template context * flow: add unit tests * conditional flow support using flow * fix proto callbacks + more unit tests * adds integration test * conditional flow: check if req has any matchers * fix lint error * deprecate iterate-all+ missing multi-proto implementation * fix ip input in raw request * JS: feat dedupe object+ more builtin funcs * feat: hide protocol result using hide * feat: async execution * complete async execution support * fix condition-flow without any matchers * refactor: template executer package (tmplexec) * flow executor working * fix data race in templateCtx * templateCtx redesign * fix failing unit test * add multiprotocol support to deprecated syntax * fix race condition in utils & tlsx * add documentation in flow package * remove regions.txt file * fix minor issue with self contained templates * fix typos of copilot * dep + misc update * fix reqID: use req.Type instead of template.Type --------- Co-authored-by: sandeep <[email protected]>
- Loading branch information
1 parent
8125b68
commit f7fe99f
Showing
78 changed files
with
2,576 additions
and
639 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
id: ghost-blog-detection | ||
info: | ||
name: Ghost blog detection | ||
author: pdteam | ||
severity: info | ||
|
||
|
||
flow: dns() && http() | ||
|
||
dns: | ||
- name: "{{FQDN}}" | ||
type: CNAME | ||
|
||
matchers: | ||
- type: word | ||
words: | ||
- "ghost.io" | ||
|
||
http: | ||
- method: GET | ||
path: | ||
- "{{BaseURL}}" | ||
|
||
matchers: | ||
- type: word | ||
words: | ||
- "ghost.io" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
id: ghost-blog-detection | ||
info: | ||
name: Ghost blog detection | ||
author: pdteam | ||
severity: info | ||
|
||
|
||
flow: dns() && http() | ||
|
||
dns: | ||
- name: "{{FQDN}}" | ||
type: CNAME | ||
|
||
matchers: | ||
- type: word | ||
words: | ||
- "ghost.io" | ||
|
||
http: | ||
- method: GET | ||
path: | ||
- "{{BaseURL}}" | ||
|
||
matchers: | ||
- type: word | ||
words: | ||
- "ghost.io" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
id: dns-ns-probe | ||
|
||
info: | ||
name: Nuclei flow dns ns probe | ||
author: pdteam | ||
severity: info | ||
description: Description of the Template | ||
reference: https://example-reference-link | ||
|
||
flow: | | ||
dns("fetch-ns"); | ||
for(let ns of template["nameservers"]) { | ||
set("nameserver",ns); | ||
dns("probe-ns"); | ||
}; | ||
dns: | ||
- id: "fetch-ns" | ||
name: "{{FQDN}}" | ||
type: NS | ||
matchers: | ||
- type: word | ||
words: | ||
- "IN\tNS" | ||
extractors: | ||
- type: regex | ||
internal: true | ||
name: "nameservers" | ||
group: 1 | ||
regex: | ||
- "IN\tNS\t(.+)" | ||
|
||
- id: "probe-ns" | ||
name: "{{nameserver}}" | ||
type: A | ||
class: inet | ||
retries: 3 | ||
recursion: true | ||
extractors: | ||
- type: dsl | ||
dsl: | ||
- "a" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
id: extract-emails | ||
|
||
info: | ||
name: Extract Email IDs from Response | ||
author: pdteam | ||
severity: info | ||
|
||
|
||
flow: | | ||
http(0) | ||
for(let email of template["emails"]) { | ||
set("email",email); | ||
http(1); | ||
} | ||
http: | ||
- method: GET | ||
path: | ||
- "{{BaseURL}}" | ||
|
||
extractors: | ||
- type: regex | ||
name: emails | ||
internal: true | ||
regex: | ||
- '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' | ||
|
||
- method: GET | ||
path: | ||
- "{{BaseURL}}/user/{{base64(email)}}" | ||
|
||
matchers: | ||
- type: word | ||
words: | ||
- "Welcome" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
id: matchet-status | ||
id: matcher-status | ||
|
||
info: | ||
name: Test Matcher Status | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,84 @@ | ||
package main | ||
|
||
import ( | ||
"encoding/base64" | ||
"fmt" | ||
"net/http" | ||
"net/http/httptest" | ||
|
||
"github.com/julienschmidt/httprouter" | ||
"github.com/projectdiscovery/nuclei/v2/pkg/testutils" | ||
) | ||
|
||
var flowTestcases = []TestCaseInfo{ | ||
{Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}}, | ||
{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, | ||
{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, | ||
{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, | ||
} | ||
|
||
type conditionalFlow struct{} | ||
|
||
func (t *conditionalFlow) Execute(filePath string) error { | ||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug) | ||
if err != nil { | ||
return err | ||
} | ||
return expectResultsCount(results, 2) | ||
} | ||
|
||
type conditionalFlowNegative struct{} | ||
|
||
func (t *conditionalFlowNegative) Execute(filePath string) error { | ||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) | ||
if err != nil { | ||
return err | ||
} | ||
return expectResultsCount(results, 0) | ||
} | ||
|
||
type iterateValuesFlow struct{} | ||
|
||
func (t *iterateValuesFlow) Execute(filePath string) error { | ||
router := httprouter.New() | ||
testemails := []string{ | ||
"[email protected]", | ||
"[email protected]", | ||
} | ||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { | ||
w.WriteHeader(http.StatusOK) | ||
_, _ = w.Write([]byte(fmt.Sprint(testemails))) | ||
}) | ||
router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { | ||
w.WriteHeader(http.StatusOK) | ||
_, _ = w.Write([]byte("Welcome ! This is test matcher text")) | ||
}) | ||
|
||
router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { | ||
w.WriteHeader(http.StatusOK) | ||
_, _ = w.Write([]byte("Welcome ! This is test matcher text")) | ||
}) | ||
|
||
ts := httptest.NewServer(router) | ||
defer ts.Close() | ||
|
||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) | ||
if err != nil { | ||
return err | ||
} | ||
return expectResultsCount(results, 2) | ||
} | ||
|
||
type dnsNsProbe struct{} | ||
|
||
func (t *dnsNsProbe) Execute(filePath string) error { | ||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug) | ||
if err != nil { | ||
return err | ||
} | ||
return expectResultsCount(results, 3) | ||
} | ||
|
||
func getBase64(input string) string { | ||
return base64.StdEncoding.EncodeToString([]byte(input)) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.