diff --git a/integration_tests/flow/conditional-flow-negative.yaml b/integration_tests/flow/conditional-flow-negative.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/integration_tests/flow/conditional-flow-negative.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/integration_tests/flow/conditional-flow.yaml b/integration_tests/flow/conditional-flow.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/integration_tests/flow/conditional-flow.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/integration_tests/flow/dns-ns-probe.yaml b/integration_tests/flow/dns-ns-probe.yaml new file mode 100644 index 0000000000..569a9e766c --- /dev/null +++ b/integration_tests/flow/dns-ns-probe.yaml @@ -0,0 +1,42 @@ +id: dns-ns-probe + +info: + name: Nuclei flow dns ns probe + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("fetch-ns"); + for(let ns of template["nameservers"]) { + set("nameserver",ns); + dns("probe-ns"); + }; + +dns: + - id: "fetch-ns" + name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - id: "probe-ns" + name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/integration_tests/flow/iterate-values-flow.yaml b/integration_tests/flow/iterate-values-flow.yaml new file mode 100644 index 0000000000..3abea9e4cb --- /dev/null +++ b/integration_tests/flow/iterate-values-flow.yaml @@ -0,0 +1,35 @@ +id: extract-emails + +info: + name: Extract Email IDs from Response + author: pdteam + severity: info + + +flow: | + http(0) + for(let email of template["emails"]) { + set("email",email); + http(1); + } + +http: + - method: GET + path: + - "{{BaseURL}}" + + extractors: + - type: regex + name: emails + internal: true + regex: + - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' + + - method: GET + path: + - "{{BaseURL}}/user/{{base64(email)}}" + + matchers: + - type: word + words: + - "Welcome" \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 0e5c9c2881..b05ccf0923 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4b0a00483046022100cd2b9d34169cdb716caee25976fed763880435f2f1e2979c9d7c9d2bd7b8e409022100dd0ba8bd3fa6a6be5f964ca3b0ce8bdbb20d865553133cf494ef64fbeebff345 \ No newline at end of file +# digest: 4a0a00473045022100e17c7a809fd64419baf401b5331edab3a68a4c182f7777614beb1862eb6ea8b7022011b95fc0e22d7f82e08e01b56ce87afdbe03027c238ba290a058d695226173ae \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 8fae1b051a..ee76c50c32 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a00483046022100f663e5afaf5c118b21b9c5918cba12d7cc83edc2a3ee0f338c07e3cd1fe40e20022100b46193e3275c490a4ad3897c6e2ca51ce09f408538b17d041e0063d40f4df833 \ No newline at end of file +# digest: 490a004630440220241d7faae14ab5760dbe7acf621a3923d0650148bc14a52a9be06ba76e8e0abf02201885fcc432d354d3c99ea97b964838719451bc97f148229f187f61eee7525eb6 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index de4abdac08..664c2d7e8f 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4b0a00483046022100c45cd27b9d49879663e1ea3c877dc362d06b8a0aea64b1ab06be3af5aa9a32ee0221008f5ee347245a2c1e04c46528e4c70a5a851f95c6ba49d2834ef7c3784bca47a9 \ No newline at end of file +# digest: 490a004630440220427cb7100f0b7d95224f490a6f4f626186782cb26c69f2551d6aefcdbc7c17d20220206161ad3a98afe8fcef9dd06d9a6dd5f34c5f7e3cd3ab7f81328f033dcd2b48 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 9b0187dd8d..6b77ca5227 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022100df57bf446d6d8e73ff9424b1055faebcea9038e5d5934834ed8e619b77bdfd5e02201754c1cebe9f65883315b3830755a0689999f33db7102cd8d5469e4c01cc6a66 \ No newline at end of file +# digest: 4a0a00473045022056092462597e85139626656d37df123094cb3861bdf583450c38814bac8df9cb022100e83a8c552f8f8a098f6b7ec8a32c6b448b995e000884beadb50cb0f2720117de \ No newline at end of file diff --git a/integration_tests/protocols/http/matcher-status.yaml b/integration_tests/protocols/http/matcher-status.yaml index 5704c2a3d9..4cfd0d1a0b 100644 --- a/integration_tests/protocols/http/matcher-status.yaml +++ b/integration_tests/protocols/http/matcher-status.yaml @@ -1,4 +1,4 @@ -id: matchet-status +id: matcher-status info: name: Test Matcher Status diff --git a/v2/cmd/integration-test/flow.go b/v2/cmd/integration-test/flow.go new file mode 100644 index 0000000000..67d4b749b4 --- /dev/null +++ b/v2/cmd/integration-test/flow.go @@ -0,0 +1,84 @@ +package main + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var flowTestcases = []TestCaseInfo{ + {Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}}, + {Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, + {Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, + {Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, +} + +type conditionalFlow struct{} + +func (t *conditionalFlow) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug) + if err != nil { + return err + } + return expectResultsCount(results, 2) +} + +type conditionalFlowNegative struct{} + +func (t *conditionalFlowNegative) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + +type iterateValuesFlow struct{} + +func (t *iterateValuesFlow) Execute(filePath string) error { + router := httprouter.New() + testemails := []string{ + "secrets@scanme.sh", + "superadmin@scanme.sh", + } + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(fmt.Sprint(testemails))) + }) + router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) + }) + + router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) + }) + + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 2) +} + +type dnsNsProbe struct{} + +func (t *dnsNsProbe) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug) + if err != nil { + return err + } + return expectResultsCount(results, 3) +} + +func getBase64(input string) string { + return base64.StdEncoding.EncodeToString([]byte(input)) +} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 66d562ccff..f18560c773 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -49,6 +49,7 @@ var ( "multi": multiProtoTestcases, "generic": genericTestcases, "dsl": dslTestcases, + "flow": flowTestcases, } // For debug purposes diff --git a/v2/cmd/integration-test/multi.go b/v2/cmd/integration-test/multi.go index a7998ab358..a9ff58fffd 100644 --- a/v2/cmd/integration-test/multi.go +++ b/v2/cmd/integration-test/multi.go @@ -1,6 +1,8 @@ package main -import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) var multiProtoTestcases = []TestCaseInfo{ {Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}}, diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 773f919254..a4e483a405 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{ //template folder path issue {Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd - {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, + {Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, //relative path {Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path diff --git a/v2/go.mod b/v2/go.mod index 0539604078..4ced433be3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,17 +21,17 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.35 - github.com/projectdiscovery/hmap v0.0.13 + github.com/projectdiscovery/fastdialer v0.0.36 + github.com/projectdiscovery/hmap v0.0.15 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.18 - github.com/projectdiscovery/retryabledns v1.0.32 - github.com/projectdiscovery/retryablehttp-go v1.0.20 + github.com/projectdiscovery/retryabledns v1.0.35 + github.com/projectdiscovery/retryablehttp-go v1.0.24 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.23.6 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 @@ -39,9 +39,9 @@ require ( github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.12.0 - golang.org/x/oauth2 v0.10.0 - golang.org/x/text v0.11.0 + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 + golang.org/x/text v0.12.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) @@ -60,6 +60,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/docker/go-units v0.5.0 + github.com/dop251/goja v0.0.0-20230812105242-81d76064690d github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 github.com/h2non/filetype v1.1.3 @@ -68,7 +69,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.12 + github.com/projectdiscovery/goflags v0.1.18 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 github.com/projectdiscovery/httpx v1.3.4 @@ -76,9 +77,9 @@ require ( github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.1 + github.com/projectdiscovery/tlsx v1.1.4 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 + github.com/projectdiscovery/utils v0.0.51 github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -87,7 +88,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect @@ -105,14 +106,16 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect - github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaukas/godicttls v0.0.3 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.8 // indirect @@ -128,11 +131,13 @@ require ( github.com/projectdiscovery/asnmap v1.0.4 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect - github.com/refraction-networking/utls v1.3.2 // indirect + github.com/quic-go/quic-go v0.37.0 // indirect + github.com/refraction-networking/utls v1.4.2 // indirect github.com/sashabaranov/go-openai v1.14.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -216,10 +221,10 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 082cecc252..efaa4f96f8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -3,8 +3,8 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= @@ -128,6 +128,9 @@ github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM2 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -145,11 +148,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= -github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20230812105242-81d76064690d h1:9aaGwVf4q+kknu+mROAXUApJ1DoOwhE8dGj/XLBYzWg= +github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -179,6 +189,7 @@ github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw4 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -190,6 +201,9 @@ github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+j github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -234,6 +248,9 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -250,8 +267,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= @@ -259,6 +276,7 @@ github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDj github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU= @@ -295,6 +313,7 @@ github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -371,8 +390,9 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= @@ -398,20 +418,20 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= -github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= -github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= +github.com/projectdiscovery/fastdialer v0.0.36 h1:Ac/CRLryJB2mA8erDwAHoCJGFvjCDIPUznxWl9kJPW8= +github.com/projectdiscovery/fastdialer v0.0.36/go.mod h1:jxX9iQJdTwlD6u0Q9Dj9/AmatHPW2GRl3V6XTAvKtHY= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0= -github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= +github.com/projectdiscovery/goflags v0.1.18 h1:L4nwDBNJcZhbmhI3GhQ1GJwz7xVWFL3BumJ+TIDBi5E= +github.com/projectdiscovery/goflags v0.1.18/go.mod h1:cZut0Q98yksNVtM73RPSm22N/eDkAMFT9t6mwu6S5pY= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6/go.mod h1:jCpXNvLUCPMzm5AhJv8wtnUt/7rz0TY2SsqvKQ8tn2E= -github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= -github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= +github.com/projectdiscovery/hmap v0.0.15 h1:iTRXV94bNIuR5obYBxOhvs3yEYXdNdJJnrXnxv4uLHc= +github.com/projectdiscovery/hmap v0.0.15/go.mod h1:oybodscUwBbL4GnhBPPTemazPXyMErqL+dE+0ZtJ6lg= github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= @@ -426,31 +446,34 @@ github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt3 github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo= -github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c= -github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE= -github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI= +github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU= +github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c= +github.com/projectdiscovery/retryablehttp-go v1.0.24 h1:1In7vIUnNvEdHhnA5KmUVf+D+tVZgITaJUZxFYgKCdo= +github.com/projectdiscovery/retryablehttp-go v1.0.24/go.mod h1:S2KiViUrjvRua/mifKEj+6Gs8TJjhFsrZwOXKJAZzSA= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= -github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= +github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoWslEpIw= +github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 h1:bgTXdrA/yFhFGfjhMIsczVNhnsMEHFidgS/FD2Tq5Js= -github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/utils v0.0.51 h1:WZV8kP4VW/I1ZkXDjyoudhhfVVHpAKKnW+Re0LVNMbc= +github.com/projectdiscovery/utils v0.0.51/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= -github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= -github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= +github.com/quic-go/quic-go v0.37.0 h1:wf/Ym2yeWi98oQn4ahiBSqdnaXVxNQGj2oBQFgiVChc= +github.com/quic-go/quic-go v0.37.0/go.mod h1:XtCUOCALTTWbPyd0IxFfHf6h0sEMubRFvEYHl3QxKw8= +github.com/refraction-networking/utls v1.4.2 h1:7N+928mSM1pEyAJb8x2Y1FbEwTIftGwn2IFykosSzwc= +github.com/refraction-networking/utls v1.4.2/go.mod h1:JkUIj+Pc8eyFB0z+A4RJRZmoT43ajjFZWVMXuZQ8BEQ= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -463,8 +486,8 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= -github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -478,8 +501,9 @@ github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2Iqp github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= @@ -619,8 +643,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -653,12 +677,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -693,6 +717,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -706,9 +731,9 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -717,19 +742,20 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -767,6 +793,7 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nR gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 8b945c4ad6..2f6b4fbff8 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.0.0` + Version = `v3.0.0-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go index d601125466..f93af3fc93 100644 --- a/v2/pkg/protocols/code/code.go +++ b/v2/pkg/protocols/code/code.go @@ -32,6 +32,8 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` // description: | // Engine type Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine,enum=python,enum=powershell,enum=command"` @@ -96,7 +98,7 @@ func (request *Request) Requests() int { // GetID returns the ID for the request if any. func (request *Request) GetID() string { - return "" + return request.ID } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. @@ -115,10 +117,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // inject all template context values as gozero env variables variables := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) + // add template context values + variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) variablesMap := request.options.Variables.Evaluate(variables) - variables = generators.MergeMaps(variablesMap, variables, optionVars) + variables = generators.MergeMaps(variablesMap, variables, optionVars, request.options.Constants) for name, value := range variables { v := fmt.Sprint(value) v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs) @@ -151,6 +155,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo + // expose response variables in proto_var format + // this is no-op if the template is not a multi protocol template + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data) + + // add variables from template context before matching/extraction + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(interactshURLs, data) } diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index aa7dfa39b5..dd594f73d1 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -50,6 +50,36 @@ func (ctx *Context) hasArgs() bool { return !ctx.args.IsEmpty() } +// Merge the key-value pairs +func (ctx *Context) Merge(args map[string]interface{}) { + _ = ctx.args.Merge(args) +} + +// Add the specific key-value pair +func (ctx *Context) Add(key string, v interface{}) { + values, ok := ctx.args.Get(key) + if !ok { + ctx.Set(key, v) + } + + // If the key exists, append the value to the existing value + switch v := v.(type) { + case []string: + if values, ok := values.([]string); ok { + values = append(values, v...) + ctx.Set(key, values) + } + case string: + if values, ok := values.(string); ok { + tmp := []string{values, v} + ctx.Set(key, tmp) + } + default: + values, _ := ctx.Get(key) + ctx.Set(key, []interface{}{values, v}) + } +} + // Get the value with specific key if exists func (ctx *Context) Get(key string) (interface{}, bool) { if !ctx.hasArgs() { @@ -86,7 +116,7 @@ func (ctx *Context) HasArgs() bool { func (ctx *Context) Clone() *Context { newCtx := &Context{ MetaInput: ctx.MetaInput.Clone(), - args: ctx.args, + args: ctx.args.Clone(), CookieJar: ctx.CookieJar, } return newCtx diff --git a/v2/pkg/protocols/common/contextargs/metainput.go b/v2/pkg/protocols/common/contextargs/metainput.go index b76603de16..4027bd7d01 100644 --- a/v2/pkg/protocols/common/contextargs/metainput.go +++ b/v2/pkg/protocols/common/contextargs/metainput.go @@ -2,6 +2,7 @@ package contextargs import ( "bytes" + "crypto/md5" "fmt" "strings" @@ -14,6 +15,8 @@ type MetaInput struct { Input string `json:"input,omitempty"` // CustomIP to use for connection CustomIP string `json:"customIP,omitempty"` + // hash of the input + hash string `json:"-"` } func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) { @@ -67,3 +70,19 @@ func (metaInput *MetaInput) PrettyPrint() string { } return metaInput.Input } + +// GetScanHash returns a unique hash that represents a scan by hashing (metainput + templateId) +func (metaInput *MetaInput) GetScanHash(templateId string) string { + // there may be some cases where metainput is changed ex: while executing self-contained template etc + // but that totally changes the scanID/hash so to avoid that we compute hash only once + // and reuse it for all subsequent calls + if metaInput.hash == "" { + metaInput.hash = getMd5Hash(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP) + } + return metaInput.hash +} + +func getMd5Hash(data string) string { + bin := md5.Sum([]byte(data)) + return string(bin[:]) +} diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go deleted file mode 100644 index 4983ae1ed7..0000000000 --- a/v2/pkg/protocols/common/executer/executer.go +++ /dev/null @@ -1,189 +0,0 @@ -package executer - -import ( - "fmt" - "strings" - "sync/atomic" - - "github.com/pkg/errors" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" -) - -// Executer executes a group of requests for a protocol -type Executer struct { - requests []protocols.Request - options *protocols.ExecutorOptions -} - -var _ protocols.Executer = &Executer{} - -// NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *Executer { - return &Executer{requests: requests, options: options} -} - -// Compile compiles the execution generators preparing any requests possible. -func (e *Executer) Compile() error { - cliOptions := e.options.Options - - for _, request := range e.requests { - if err := request.Compile(e.options); err != nil { - var dslCompilationError *dsl.CompilationError - if errors.As(err, &dslCompilationError) { - if cliOptions.Verbose { - rawErrorMessage := dslCompilationError.Error() - formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "." - gologger.Warning().Msgf(formattedErrorMessage) - gologger.Info().Msgf("The available custom DSL functions are:") - fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor)) - } - } - return err - } - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int { - var count int - for _, request := range e.requests { - count += request.Requests() - } - return count -} - -// Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input *contextargs.Context) (bool, error) { - results := &atomic.Bool{} - - dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) { - dynamicValues[key] = value - }) - } - previous := make(map[string]interface{}) - - var lastMatcherEvent *output.InternalWrappedEvent - writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { - if !results.Load() && matcherStatus { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } - results.CompareAndSwap(false, true) - } - } - - for _, req := range e.requests { - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" { - return false, nil - } - } - - err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { - if event == nil { - // ideally this should never happen since protocol exits on error and callback is not called - return - } - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - previous[builder.String()] = v - builder.Reset() - } - } - // If no results were found, and also interactsh is not being used - // in that case we can skip it, otherwise we've to show failure in - // case of matcher-status flag. - if !event.HasOperatorResult() && !event.UsesInteractsh { - lastMatcherEvent = event - } else { - if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { - results.CompareAndSwap(false, true) - } else { - lastMatcherEvent = event - } - } - }) - if err != nil { - if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) - } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) - } - // If a match was found and stop at first match is set, break out of the loop and return - if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - break - } - } - if lastMatcherEvent != nil { - writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) - } - return results.Load(), nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { - dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) { - dynamicValues[key] = value - }) - } - previous := make(map[string]interface{}) - results := &atomic.Bool{} - - for _, req := range e.requests { - req := req - - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" { - return nil - } - } - - err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - previous[builder.String()] = v - builder.Reset() - } - } - if event.OperatorsResult == nil { - return - } - results.CompareAndSwap(false, true) - callback(event) - }) - if err != nil { - if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) - } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) - } - // If a match was found and stop at first match is set, break out of the loop and return - if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - break - } - } - return nil -} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 98fc6aa138..c17fef84c8 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // PayloadGenerator is the generator struct for generating payloads @@ -14,10 +14,11 @@ type PayloadGenerator struct { Type AttackType catalog catalog.Catalog payloads map[string][]string + options *types.Options } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, allowLocalFileAccess bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog, customAttackType string, opts *types.Options) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -38,12 +39,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } } - generator := &PayloadGenerator{catalog: catalog} + generator := &PayloadGenerator{catalog: catalog, options: opts} if err := generator.validate(payloadsFinal, templatePath); err != nil { return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, allowLocalFileAccess) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 7187808904..2226188dca 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -6,13 +6,14 @@ import ( "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -32,7 +33,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -54,7 +55,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -83,3 +84,9 @@ func TestClusterbombGenerator(t *testing.T) { } require.Equal(t, 3, count, "could not get correct clusterbomb counts") } + +func getOptions(allowLocalFileAccess bool) *types.Options { + opts := types.DefaultOptions() + opts.AllowLocalFileAccess = allowLocalFileAccess + return opts +} diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 92ec932019..390c8141e4 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,7 +2,7 @@ package generators import ( "bufio" - "path/filepath" + "io" "strings" "github.com/pkg/errors" @@ -11,7 +11,7 @@ import ( ) // loadPayloads loads the input payloads from a map to a data map -func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, allowLocalFileAccess bool) (map[string][]string, error) { +func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath string) (map[string][]string, error) { loadedPayloads := make(map[string][]string) for name, payload := range payloads { @@ -22,18 +22,11 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, if len(elements) >= 2 { loadedPayloads[name] = elements } else { - if !allowLocalFileAccess { - pt = filepath.Clean(pt) - templateAbsPath, err := filepath.Abs(templatePath) - if err != nil { - return nil, errors.Wrap(err, "could not get absolute path") - } - templatePathDir := filepath.Dir(templateAbsPath) - if !(templatePathDir != "/" && strings.HasPrefix(pt, templatePathDir)) && !strings.HasPrefix(pt, templateDirectory) { - return nil, errors.New("denied payload file path specified") - } + file, err := generator.options.LoadHelperFile(pt, templatePath, generator.catalog) + if err != nil { + return nil, errors.Wrap(err, "could not load payload file") } - payloads, err := generator.loadPayloadsFromFile(pt) + payloads, err := generator.loadPayloadsFromFile(file) if err != nil { return nil, errors.Wrap(err, "could not load payloads") } @@ -47,13 +40,8 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, } // loadPayloadsFromFile loads a file to a string slice -func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) { +func (generator *PayloadGenerator) loadPayloadsFromFile(file io.ReadCloser) ([]string, error) { var lines []string - - file, err := generator.catalog.OpenFile(filepath) - if err != nil { - return nil, err - } defer file.Close() scanner := bufio.NewScanner(file) diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index 28803b097b..49c00ef2f9 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -2,66 +2,119 @@ package generators import ( "os" + "os/exec" "path/filepath" "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" osutils "github.com/projectdiscovery/utils/os" "github.com/stretchr/testify/require" ) func TestLoadPayloads(t *testing.T) { - tempdir, err := os.MkdirTemp("", "templates-*") - require.NoError(t, err, "could not create temp dir") - defer os.RemoveAll(tempdir) + // since we are changing value of global variable i.e templates directory + // run this test as subprocess + if os.Getenv("LOAD_PAYLOAD_NO_ACCESS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess") + cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_NO_ACCESS=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + if err != nil { + t.Fatalf("process ran with err %v, want exit status 1", err) + } + } + templateDir := getTemplatesDir(t) + config.DefaultConfig.SetTemplatesDir(templateDir) - generator := &PayloadGenerator{catalog: disk.NewCatalog(tempdir)} - - fullpath := filepath.Join(tempdir, "payloads.txt") - err = os.WriteFile(fullpath, []byte("test\nanother"), 0777) - require.NoError(t, err, "could not write payload") + generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(false)} + fullpath := filepath.Join(templateDir, "payloads.txt") // Test sandbox t.Run("templates-directory", func(t *testing.T) { + // testcase when loading file from template directory and template file is in root + // expected to succeed values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/test", tempdir, false) + }, "/test") require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) t.Run("templates-path-relative", func(t *testing.T) { + // testcase when loading file from template directory and template file is current working directory + // expected to fail since this is LFI _, err := generator.loadPayloads(map[string]interface{}{ "new": "../../../../../../../../../etc/passwd", - }, ".", tempdir, false) + }, ".") require.Error(t, err, "could load payloads") }) t.Run("template-directory", func(t *testing.T) { + // testcase when loading file from template directory and template file is inside template directory + // expected to succeed values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, filepath.Join(tempdir, "test.yaml"), "/test", false) + }, filepath.Join(templateDir, "test.yaml")) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) - t.Run("no-sandbox-unix", func(t *testing.T) { - if osutils.IsWindows() { - return - } - _, err := generator.loadPayloads(map[string]interface{}{ - "new": "/etc/passwd", - }, "/random", "/test", true) - require.NoError(t, err, "could load payloads") - }) + t.Run("invalid", func(t *testing.T) { + // testcase when loading file from /etc/passwd and template file is at root i.e / + // expected to fail since this is LFI values, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", false) + }, "/random") require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") + // testcase when loading file from template directory and template file is at root i.e / + // expected to succeed values, err = generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/random", "/test", false) - require.Error(t, err, "could load payloads") - require.Equal(t, 0, len(values), "could get values") + }, "/random") + require.NoError(t, err, "could load payloads %v", values) + require.Equal(t, 1, len(values), "could get values") + require.Equal(t, []string{"test", "another"}, values["new"], "could get values") + }) +} + +func TestLoadPayloadsWithAccess(t *testing.T) { + // since we are changing value of global variable i.e templates directory + // run this test as subprocess + if os.Getenv("LOAD_PAYLOAD_WITH_ACCESS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess") + cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_WITH_ACCESS=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + if err != nil { + t.Fatalf("process ran with err %v, want exit status 1", err) + } + } + templateDir := getTemplatesDir(t) + config.DefaultConfig.SetTemplatesDir(templateDir) + + generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(true)} + + t.Run("no-sandbox-unix", func(t *testing.T) { + if osutils.IsWindows() { + return + } + _, err := generator.loadPayloads(map[string]interface{}{ + "new": "/etc/passwd", + }, "/random") + require.NoError(t, err, "could load payloads") }) } + +func getTemplatesDir(t *testing.T) string { + tempdir, err := os.MkdirTemp("", "templates-*") + require.NoError(t, err, "could not create temp dir") + fullpath := filepath.Join(tempdir, "payloads.txt") + err = os.WriteFile(fullpath, []byte("test\nanother"), 0777) + require.NoError(t, err, "could not write payload") + return tempdir +} diff --git a/v2/pkg/protocols/common/variables/variables.go b/v2/pkg/protocols/common/variables/variables.go index b806402aaa..7a7fda1f03 100644 --- a/v2/pkg/protocols/common/variables/variables.go +++ b/v2/pkg/protocols/common/variables/variables.go @@ -66,6 +66,11 @@ func (variables *Variable) UnmarshalJSON(data []byte) error { func (variables *Variable) Evaluate(values map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}, variables.Len()) variables.ForEach(func(key string, value interface{}) { + if sliceValue, ok := value.([]interface{}); ok { + // slices cannot be evaluated + result[key] = sliceValue + return + } valueString := types.ToString(value) combined := generators.MergeMaps(values, result) if value, ok := combined[key]; ok { @@ -76,12 +81,26 @@ func (variables *Variable) Evaluate(values map[string]interface{}) map[string]in return result } +// GetAll returns all variables as a map +func (variables *Variable) GetAll() map[string]interface{} { + result := make(map[string]interface{}, variables.Len()) + variables.ForEach(func(key string, value interface{}) { + result[key] = value + }) + return result +} + // EvaluateWithInteractsh returns evaluation results of variables with interactsh func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{}, interact *interactsh.Client) (map[string]interface{}, []string) { result := make(map[string]interface{}, variables.Len()) var interactURLs []string variables.ForEach(func(key string, value interface{}) { + if sliceValue, ok := value.([]interface{}); ok { + // slices cannot be evaluated + result[key] = sliceValue + return + } valueString := types.ToString(value) if strings.Contains(valueString, "interactsh-url") { valueString, interactURLs = interact.Replace(valueString, interactURLs) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 1cc8302e57..232427058a 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -172,7 +172,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 8d05d37172..c2e1fcdbed 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) // merge with metadata (eg. from workflow context) - vars = generators.MergeMaps(vars, metadata, optionVars, request.options.TemplateCtx.GetAll()) + vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(vars, variablesMap, request.options.Constants) @@ -66,18 +66,18 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, break } value = generators.MergeMaps(vars, value) - if err := request.execute(domain, metadata, previous, value, callback); err != nil { + if err := request.execute(input, domain, metadata, previous, value, callback); err != nil { return err } } } else { value := maps.Clone(vars) - return request.execute(domain, metadata, previous, value, callback) + return request.execute(input, domain, metadata, previous, value, callback) } return nil } -func (request *Request) execute(domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { +func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { if vardump.EnableVarDump { gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) @@ -151,7 +151,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData) // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) for k, v := range previous { outputEvent[k] = v } @@ -159,7 +159,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent[k] = v } // add variables from template context before matching/extraction - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) dumpResponse(event, request, request.options, response.String(), question) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index dc19deccf6..324b5a1946 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -64,7 +64,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // every new file in the compressed multi-file archive counts 1 request.options.Progress.AddToTotal(1) archiveFileName := filepath.Join(filePath, file.Name()) - event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous) + event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -117,7 +117,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, _ = tmpFileOut.Sync() // rewind the file _, _ = tmpFileOut.Seek(0, 0) - event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous) + event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -137,7 +137,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, default: // normal file - increments the counter by 1 request.options.Progress.AddToTotal(1) - event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous) + event, fileMatches, err := request.processFile(filePath, input, previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -166,7 +166,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, return nil } -func (request *Request) processFile(filePath, input string, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { +func (request *Request) processFile(filePath string, input *contextargs.Context, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { file, err := os.Open(filePath) if err != nil { return nil, nil, errors.Errorf("Could not open file path %s: %s\n", filePath, err) @@ -185,7 +185,7 @@ func (request *Request) processFile(filePath, input string, previousInternalEven return request.processReader(file, filePath, input, stat.Size(), previousInternalEvent) } -func (request *Request) processReader(reader io.Reader, filePath, input string, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { +func (request *Request) processReader(reader io.Reader, filePath string, input *contextargs.Context, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { fileReader := io.LimitReader(reader, request.maxSize) fileMatches, opResult := request.findMatchesWithReader(fileReader, input, filePath, totalBytes, previousInternalEvent) if opResult == nil && len(fileMatches) == 0 { @@ -193,10 +193,10 @@ func (request *Request) processReader(reader io.Reader, filePath, input string, } // build event structure to interface with internal logic - return request.buildEvent(input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil + return request.buildEvent(input.MetaInput.Input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil } -func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) { +func (request *Request) findMatchesWithReader(reader io.Reader, input *contextargs.Context, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) { var bytesCount, linesCount, wordsCount int isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse totalBytesString := units.BytesSize(float64(totalBytes)) @@ -243,12 +243,12 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath processedBytes := units.BytesSize(float64(currentBytes)) gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytesString) - dslMap := request.responseToDSLMap(lineContent, input, filePath) + dslMap := request.responseToDSLMap(lineContent, input.MetaInput.Input, filePath) for k, v := range previous { dslMap[k] = v } // add template context variables to DSL map - dslMap = generators.MergeMaps(dslMap, request.options.TemplateCtx.GetAll()) + dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll()) discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug) newOpResult := discardEvent.OperatorsResult if newOpResult != nil { diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3524b47d86..7930598d21 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -349,7 +349,7 @@ func TestActionGetResource(t *testing.T) { Nuclei Test Page - + ` @@ -360,7 +360,7 @@ func TestActionGetResource(t *testing.T) { testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") - require.Equal(t, len(out["src"]), 3159, "could not find resource") + require.Equal(t, len(out["src"]), 121808, "could not find resource") }) } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 0d1d09e5e7..32dc1af8f0 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -91,6 +91,8 @@ func (request *Request) GetID() string { // Compile compiles the protocol request for further execution. func (request *Request) Compile(options *protocols.ExecutorOptions) error { + request.options = options + // TODO: logic similar to network + http => probably can be refactored // Resolve payload paths from vars if they exists for name, payload := range options.Options.Vars.AsMap() { @@ -106,7 +108,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { if len(request.Payloads) > 0 { var err error - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Catalog, options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } @@ -136,7 +138,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.CompiledOperators = compiled } - request.options = options if len(request.Fuzzing) > 0 { for _, rule := range request.Fuzzing { diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 049cfb5a53..24aec77abb 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -44,7 +44,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, vars := protocolutils.GenerateVariablesWithContextArgs(input, false) payloads := generators.BuildPayloadFromOptions(request.options.Options) // add templatecontext variables to varMap - values := generators.MergeMaps(vars, metadata, payloads, request.options.TemplateCtx.GetAll()) + values := generators.MergeMaps(vars, metadata, payloads, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(values) payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants) @@ -156,8 +156,8 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory()) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) for k, v := range out { outputEvent[k] = v } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 13a45f7cd0..69661e3dbd 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -72,7 +72,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, // 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path // add template context values to dynamicValues (this takes care of self-contained and other types of requests) - dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll()) + // Note: `iterate-all` and flow are mutually exclusive. flow uses templateCtx and iterate-all uses dynamicValues + dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.GetTemplateCtx(input.MetaInput).GetAll()) if r.request.SelfContained { return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues) } @@ -85,7 +86,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, } } else { for payloadName, payloadValue := range payloads { - payloads[payloadName] = types.ToString(payloadValue) + payloads[payloadName] = types.ToStringNSlice(payloadValue) } } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 552d9229bd..afd5545a17 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -190,6 +190,7 @@ type Request struct { SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" json:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` // description: | // IterateAll iterates all the values extracted from internal extractors + // Deprecated: Use flow instead . iterate-all will be removed in future releases IterateAll bool `yaml:"iterate-all,omitempty" json:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"` // description: | // DigestAuthUsername specifies the username for digest authentication @@ -353,7 +354,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 58aae6cb82..c65ffe38b6 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -102,7 +102,10 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b if _, ok := rawrequest.Headers["Host"]; !ok { rawrequest.Headers["Host"] = inputURL.Host } - rawrequest.FullURL = fmt.Sprintf("%s://%s%s", inputURL.Scheme, strings.TrimSpace(inputURL.Host), rawrequest.Path) + cloned := inputURL.Clone() + cloned.Path = "" + _ = cloned.MergePath(rawrequest.Path, true) + rawrequest.FullURL = cloned.String() } return rawrequest, nil diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 7e5f40040d..79a43a7d03 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -721,8 +721,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 89447b9936..c57291d584 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestRequestGeneratorPaths(t *testing.T) { @@ -34,7 +35,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance, "", types.DefaultOptions()) require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) @@ -58,7 +59,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance, "", types.DefaultOptions()) require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) diff --git a/v2/pkg/protocols/multi/doc.go b/v2/pkg/protocols/multi/doc.go deleted file mode 100644 index ec688892c2..0000000000 --- a/v2/pkg/protocols/multi/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -package multi - -// multi is a wrapper protocol Request that allows multiple protocols requests to be executed -// multi protocol is just a wrapper so it should/does not include any protocol specific code - - diff --git a/v2/pkg/protocols/multi/request.go b/v2/pkg/protocols/multi/request.go deleted file mode 100644 index 4f092c0f8f..0000000000 --- a/v2/pkg/protocols/multi/request.go +++ /dev/null @@ -1,172 +0,0 @@ -package multi - -import ( - "strconv" - - "github.com/projectdiscovery/nuclei/v2/pkg/model" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" - errorutil "github.com/projectdiscovery/utils/errors" -) - -var _ protocols.Request = &Request{} - -// refer doc.go for package description , limitations etc - -// Request contains a multi protocol request -type Request struct { - // description: | - // ID is the unique id for the template. - // - // #### Good IDs - // - // A good ID uniquely identifies what the requests in the template - // are doing. Let's say you have a template that identifies a git-config - // file on the webservers, a good name would be `git-config-exposure`. Another - // example name is `azure-apps-nxdomain-takeover`. - // examples: - // - name: ID Example - // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` - // description: | - // Info contains metadata information about the template. - // examples: - // - value: exampleInfoStructure - Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` - - // Queue is queue of all protocols present in the template - Queue []protocols.Request `yaml:"-" json:"-"` - // request executor options - options *protocols.ExecutorOptions `yaml:"-" json:"-"` -} - -// getLastRequest returns the last request in the queue -func (r *Request) getLastRequest() protocols.Request { - if len(r.Queue) == 0 { - return nil - } - return r.Queue[len(r.Queue)-1] -} - -// Requests returns the total number of requests template will send -func (r *Request) Requests() int { - var count int - for _, protocol := range r.Queue { - count += protocol.Requests() - } - return count -} - -// Compile compiles the protocol request for further execution. -func (r *Request) Compile(executerOptions *protocols.ExecutorOptions) error { - r.options = executerOptions - r.options.TemplateCtx = contextargs.New() - r.options.ProtocolType = types.MultiProtocol - for _, protocol := range r.Queue { - if err := protocol.Compile(r.options); err != nil { - return errorutil.NewWithErr(err).Msgf("failed to compile protocol %s", protocol.Type()) - } - } - return nil -} - -// GetID returns the unique template ID -func (r *Request) GetID() string { - return r.ID -} - -// Match executes matcher on model and returns true or false (used for clustering if request supports clustering) -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - return protocols.MakeDefaultMatchFunc(data, matcher) -} - -// Extract performs extracting operation for an extractor on model and returns true or false (used for clustering if request supports clustering) -func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { - return protocols.MakeDefaultExtractFunc(data, matcher) -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var finalProtoEvent *output.InternalWrappedEvent - // callback to process results from all protocols - multiProtoCallback := func(event *output.InternalWrappedEvent) { - finalProtoEvent = event - // export dynamic values from operators (i.e internal:true) - if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 { - for k, v := range event.OperatorsResult.DynamicValues { - // TBD: iterate-all is only supported in `http` protocol - // we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context) - // currently if dynamic value array only contains one value we replace it with the value - if len(v) == 1 { - r.options.TemplateCtx.Set(k, v[0]) - } else { - // Note: if extracted value contains multiple values then they can be accessed by indexing - // ex: if values are dynamic = []string{"a","b","c"} then they are available as - // dynamic = "a" , dynamic1 = "b" , dynamic2 = "c" - // we intentionally omit first index for unknown situations (where no of extracted values are not known) - for i, val := range v { - if i == 0 { - r.options.TemplateCtx.Set(k, val) - } else { - r.options.TemplateCtx.Set(k+strconv.Itoa(i), val) - } - } - } - } - } - } - - // template context: contains values extracted using `internal` extractor from previous protocols - // these values are extracted from each protocol in queue and are passed to next protocol in queue - // instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows) - // this makes it possible to use multi protocol templates in workflows - // Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow) - - // execute all protocols in the queue - for _, req := range r.Queue { - err := req.ExecuteWithResults(input, dynamicValues, previous, multiProtoCallback) - // if error skip execution of next protocols - if err != nil { - return err - } - } - // Review: how to handle events of multiple protocols in a single template - // currently the outer callback is only executed once (for the last protocol in queue) - // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executer.go#L150 - // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables - callback(finalProtoEvent) - - return nil -} - -// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally -func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { - if r.getLastRequest() == nil { - return nil - } - return r.getLastRequest().MakeResultEventItem(wrapped) -} - -// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - return protocols.MakeDefaultResultEvent(r.getLastRequest(), wrapped) -} - -// GetCompiledOperators returns a list of the compiled operators -func (r *Request) GetCompiledOperators() []*operators.Operators { - last := r.getLastRequest() - if last == nil { - return nil - } - return last.GetCompiledOperators() -} - -// Type returns the type of the protocol request -func (r *Request) Type() types.ProtocolType { - return types.MultiProtocol -} diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index c344538f58..7a5795ac97 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 744f4fd6dc..ad76e19c3d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -56,7 +56,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } variables := protocolutils.GenerateVariables(address, false, nil) // add template ctx variables to varMap - variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll()) + variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(variables) variables = generators.MergeMaps(variablesMap, variables, request.options.Constants) @@ -70,7 +70,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } visitedAddresses.Set(actualAddress, struct{}{}) - if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { outputEvent := request.responseToDSLMap("", "", "", address, "") callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err) @@ -81,7 +81,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } // executeAddress executes the request for an address -func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address}) payloads := generators.BuildPayloadFromOptions(request.options.Options) @@ -114,7 +114,7 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA return nil } -func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn @@ -279,10 +279,10 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac } response := responseBuilder.String() - outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) + outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input.MetaInput.Input, actualAddress) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) outputEvent["ip"] = request.dialer.GetDialedIP(hostname) if request.options.StopAtFirstMatch { outputEvent["stop-at-first-match"] = true diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 106023e9ee..a942288aff 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -88,8 +88,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 6f7c3bf88a..b894d7a164 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,7 +1,10 @@ package protocols import ( + "sync/atomic" + "github.com/projectdiscovery/ratelimit" + mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" "github.com/logrusorgru/aurora" @@ -86,46 +89,87 @@ type ExecutorOptions struct { Colorizer aurora.Aurora WorkflowLoader model.WorkflowLoader ResumeCfg *types.ResumeCfg - // TemplateContext (contains all variables that are templatescoped i.e multi protocol) - // only used in case of multi protocol templates - TemplateCtx *contextargs.Context // ProtocolType is the type of the template ProtocolType templateTypes.ProtocolType + // Flow is execution flow for the template (written in javascript) + Flow string + // IsMultiProtocol is true if template has more than one protocol + IsMultiProtocol bool + // templateStore is a map which contains template context for each scan (i.e input * template-id pair) + templateCtxStore *mapsutil.SyncLockMap[string, *contextargs.Context] +} + +// CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan) +func (e *ExecutorOptions) CreateTemplateCtxStore() { + e.templateCtxStore = &mapsutil.SyncLockMap[string, *contextargs.Context]{ + Map: make(map[string]*contextargs.Context), + ReadOnly: atomic.Bool{}, + } +} + +// RemoveTemplateCtx removes template context of given scan from store +func (e *ExecutorOptions) RemoveTemplateCtx(input *contextargs.MetaInput) { + scanId := input.GetScanHash(e.TemplateID) + if e.templateCtxStore != nil { + e.templateCtxStore.Delete(scanId) + } +} + +// GetTemplateCtx returns template context for given input +func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contextargs.Context { + scanId := input.GetScanHash(e.TemplateID) + templateCtx, ok := e.templateCtxStore.Get(scanId) + if !ok { + // if template context does not exist create new and add it to store and return it + templateCtx = contextargs.New() + _ = e.templateCtxStore.Set(scanId, templateCtx) + } + return templateCtx } // AddTemplateVars adds vars to template context with given template type as prefix // this method is no-op if template is not multi protocol -func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) { - if e.ProtocolType != templateTypes.MultiProtocol { - // no-op if not multi protocol template +func (e *ExecutorOptions) AddTemplateVars(input *contextargs.MetaInput, reqType templateTypes.ProtocolType, reqID string, vars map[string]interface{}) { + // if we wan't to disable adding response variables and other variables to template context + // this is the statement that does it . template context is currently only enabled for + // multiprotocol and flow templates + if !e.IsMultiProtocol && e.Flow == "" { + // no-op if not multi protocol template or flow template return } + templateCtx := e.GetTemplateCtx(input) for k, v := range vars { if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") { - if templateType < templateTypes.InvalidProtocol { - k = templateType.String() + "_" + k + if reqID != "" { + k = reqID + "_" + k + } else if reqType < templateTypes.InvalidProtocol { + k = reqType.String() + "_" + k } - e.TemplateCtx.Set(k, v) + templateCtx.Set(k, v) } } } // AddTemplateVar adds given var to template context with given template type as prefix // this method is no-op if template is not multi protocol -func (e *ExecutorOptions) AddTemplateVar(prefix, key string, value interface{}) { - if e.ProtocolType != templateTypes.MultiProtocol { - // no-op if not multi protocol template +func (e *ExecutorOptions) AddTemplateVar(input *contextargs.MetaInput, templateType templateTypes.ProtocolType, reqID string, key string, value interface{}) { + if !e.IsMultiProtocol && e.Flow == "" { + // no-op if not multi protocol template or flow template return } - if prefix != "" { - key = prefix + "_" + key + templateCtx := e.GetTemplateCtx(input) + if reqID != "" { + key = reqID + "_" + key + } else if templateType < templateTypes.InvalidProtocol { + key = templateType.String() + "_" + key } - e.TemplateCtx.Set(key, value) + templateCtx.Set(key, value) } // Copy returns a copy of the executeroptions structure func (e ExecutorOptions) Copy() ExecutorOptions { copy := e + copy.CreateTemplateCtxStore() return copy } diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index b13c3030cd..dd8b1faf3f 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -41,6 +41,9 @@ type Request struct { operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-" json:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` + // description: | // Address contains address for the request Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"` @@ -184,7 +187,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa hostnameVariables := protocolutils.GenerateDNSVariables(hostname) // add template context variables to varMap - values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll()) + values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(values) payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants) @@ -219,7 +222,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err) - gologger.Verbose().Msgf("Sent SSL request to %s", hostPort) + gologger.Verbose().Msgf("[%s] Sent SSL request to %s", request.options.TemplateID, hostPort) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.MetaInput.Input) @@ -267,7 +270,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if tag == "" || f.IsZero() { continue } - request.options.AddTemplateVar(request.Type().String(), tag, f.Value()) + request.options.AddTemplateVar(input.MetaInput, request.Type(), request.ID, tag, f.Value()) data[tag] = f.Value() } @@ -286,12 +289,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if tag == "" || f.IsZero() { continue } - request.options.AddTemplateVar(request.Type().String(), tag, f.Value()) + request.options.AddTemplateVar(input.MetaInput, request.Type(), request.ID, tag, f.Value()) data[tag] = f.Value() } // add response fields ^ to template context and merge templatectx variables to output event - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index a2d811eef1..4e8481a007 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -42,6 +42,8 @@ type Request struct { operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-" json:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` // description: | // Address contains address for the request Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` @@ -106,7 +108,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions()) if err != nil { return errors.Wrap(err, "could not parse payloads") } @@ -152,13 +154,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if !ok { break } - if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { return err } } } else { value := make(map[string]interface{}) - if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { return err } } @@ -166,8 +168,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(target *contextargs.Context, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { header := http.Header{} + input := target.MetaInput.Input parsed, err := urlutil.Parse(input) if err != nil { @@ -176,7 +179,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam defaultVars := protocolutils.GenerateVariables(parsed, false, nil) optionVars := generators.BuildPayloadFromOptions(request.options.Options) // add templatecontext variables to varMap - variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll())) + variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(target.MetaInput).GetAll())) payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants) requestOptions := request.options @@ -265,8 +268,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["ip"] = request.dialer.GetDialedIP(hostname) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), data) - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(target.MetaInput, request.Type(), request.ID, data) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(target.MetaInput).GetAll()) for k, v := range previous { data[k] = v diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 5d538ee095..ac9ae7828b 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -34,6 +34,9 @@ type Request struct { operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-" json:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` + // description: | // Query contains query for the request Query string `yaml:"query,omitempty" json:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"` @@ -91,7 +94,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa defaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) optionVars := generators.BuildPayloadFromOptions(request.options.Options) // add templatectx variables to varMap - vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll())) + vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(input.MetaInput).GetAll())) variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants) @@ -134,8 +137,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["response"] = jsonDataString // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), data) - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 954c4960cc..0b2d643ca3 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -3,20 +3,23 @@ package templates import ( "fmt" "io" + "path/filepath" "reflect" + "strings" "github.com/pkg/errors" "gopkg.in/yaml.v2" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -59,7 +62,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo } defer reader.Close() options.TemplatePath = filePath - template, err := ParseTemplateFromReader(reader, preprocessor, options) + template, err := ParseTemplateFromReader(reader, preprocessor, options.Copy()) if err != nil { return nil, err } @@ -124,34 +127,43 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti var requests []protocols.Request - if len(template.MultiProtoRequest.Queue) > 0 { - template.MultiProtoRequest.ID = template.ID - template.MultiProtoRequest.Info = template.Info - requests = append(requests, &template.MultiProtoRequest) + if template.hasMultipleRequests() { + // when multiple requests are present preserve the order of requests and protocols + // which is already done during unmarshalling + requests = template.RequestsQueue + if options.Flow == "" { + options.IsMultiProtocol = true + } } else { - switch { - case len(template.RequestsDNS) > 0: + if len(template.RequestsDNS) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) - case len(template.RequestsFile) > 0: + } + if len(template.RequestsFile) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) - case len(template.RequestsNetwork) > 0: + } + if len(template.RequestsNetwork) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) - case len(template.RequestsHTTP) > 0: + } + if len(template.RequestsHTTP) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) - case len(template.RequestsHeadless) > 0 && options.Options.Headless: + } + if len(template.RequestsHeadless) > 0 && options.Options.Headless { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) - case len(template.RequestsSSL) > 0: + } + if len(template.RequestsSSL) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) - case len(template.RequestsWebsocket) > 0: + } + if len(template.RequestsWebsocket) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) - case len(template.RequestsWHOIS) > 0: + } + if len(template.RequestsWHOIS) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) } + if len(template.RequestsCode) > 0 { + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...) + } } - if len(template.RequestsCode) > 0 { - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...) - } - template.Executer = executer.NewExecuter(requests, &options) + template.Executer = tmplexec.NewTemplateExecuter(requests, &options) return nil } @@ -196,7 +208,7 @@ mainLoop: } if len(operatorsList) > 0 { options.Operators = operatorsList - template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) return nil } @@ -237,10 +249,36 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option options.Variables = template.Variables } + // if more than 1 request per protocol exist we add request id to protocol request + // since in template context we have proto_prefix for each protocol it is overwritten + // if request id is not present + template.validateAllRequestIDs() + + // TODO: we should add a syntax check here or somehow use a javascript linter + // simplest option for now seems to compile using goja and see if it fails + if strings.TrimSpace(template.Flow) != "" { + if len(template.Flow) > 0 && filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) { + // load file respecting sandbox + file, err := options.Options.LoadHelperFile(template.Flow, options.TemplatePath, options.Catalog) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("loading flow file from %v denied", template.Flow) + } + defer file.Close() + if bin, err := io.ReadAll(file); err == nil { + template.Flow = string(bin) + } else { + return nil, errorutil.NewWithErr(err).Msgf("something went wrong failed to read file") + } + } + options.Flow = template.Flow + } + // create empty context args for template scope - options.TemplateCtx = contextargs.New() + options.CreateTemplateCtxStore() options.ProtocolType = template.Type() options.Constants = template.Constants + + template.Options = &options // If no requests, and it is also not a workflow, return error. if template.Requests() == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) diff --git a/v2/pkg/templates/compile_test.go b/v2/pkg/templates/compile_test.go index 091dc07a9f..4e67918e50 100644 --- a/v2/pkg/templates/compile_test.go +++ b/v2/pkg/templates/compile_test.go @@ -25,7 +25,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/projectdiscovery/ratelimit" @@ -197,16 +196,3 @@ func Test_WrongTemplate(t *testing.T) { require.Nil(t, got, "could not parse template") require.ErrorContains(t, err, "no requests defined ") } - -func Test_Multiprotocol(t *testing.T) { - setup() - got, err := templates.Parse("tests/multiproto.yaml", nil, executerOpts) - require.Nil(t, err, "could not parse template") - require.Equal(t, 3, got.Requests()) - require.Equal(t, types.MultiProtocol, got.Type()) - - got, err = templates.Parse("tests/multiproto.json", nil, executerOpts) - require.Nil(t, err, "could not parse template") - require.Equal(t, 3, got.Requests()) - require.Equal(t, types.MultiProtocol, got.Type()) -} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b870306edc..72a3cb069b 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -3,6 +3,7 @@ package templates import ( "encoding/json" + "strconv" validate "github.com/go-playground/validator/v10" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -13,7 +14,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/multi" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" @@ -47,6 +47,18 @@ type Template struct { // - value: exampleInfoStructure Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` // description: | + // Flow contains the execution flow for the template. + // examples: + // - flow: | + // for region in regions { + // http(0) + // } + // for vpc in vpcs { + // http(1) + // } + // + Flow string `yaml:"flow,omitempty" json:"flow,omitempty" jsonschema:"title=template execution flow in js,description=Flow contains js code which defines how the template should be executed"` + // description: | // Requests contains the http request to make in the template. // WARNING: 'requests' will be deprecated and will be removed in a future release. Please use 'http' instead. // examples: @@ -134,15 +146,13 @@ type Template struct { // Verified defines if the template signature is digitally verified Verified bool `yaml:"-" json:"-"` - // MultiProtoRequest (Internal) contains multi protocol request if multiple protocols are used - MultiProtoRequest multi.Request `yaml:"-" json:"-"` + // RequestsQueue contains all template requests in order (both protocol & request order) + RequestsQueue []protocols.Request `yaml:"-" json:"-"` } // Type returns the type of the template func (template *Template) Type() types.ProtocolType { switch { - case len(template.MultiProtoRequest.Queue) > 0: - return types.MultiProtocol case len(template.RequestsDNS) > 0: return types.DNSProtocol case len(template.RequestsFile) > 0: @@ -168,9 +178,84 @@ func (template *Template) Type() types.ProtocolType { } } +// validateAllRequestIDs check if that protocol already has given id if not +// then is is manually set to proto_index +func (template *Template) validateAllRequestIDs() { + // this is required in multiprotocol and flow where we save response variables + // and all other data in template context if template as two requests in a protocol + // then it is overwritten to avoid this we use proto_index as request ID + if len(template.RequestsCode) > 1 { + for i, req := range template.RequestsCode { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsDNS) > 1 { + for i, req := range template.RequestsDNS { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsFile) > 1 { + for i, req := range template.RequestsFile { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsHTTP) > 1 { + for i, req := range template.RequestsHTTP { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsHeadless) > 1 { + for i, req := range template.RequestsHeadless { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + + } + if len(template.RequestsNetwork) > 1 { + for i, req := range template.RequestsNetwork { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsSSL) > 1 { + for i, req := range template.RequestsSSL { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsWebsocket) > 1 { + for i, req := range template.RequestsWebsocket { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsWHOIS) > 1 { + for i, req := range template.RequestsWHOIS { + if req.ID == "" { + req.ID = req.Type().String() + "_" + strconv.Itoa(i) + } + } + } +} + // MarshalYAML forces recursive struct validation during marshal operation func (template *Template) MarshalYAML() ([]byte, error) { out, marshalErr := yaml.Marshal(template) + // Review: we are adding requestIDs for templateContext + // if we are using this method then we might need to purge manually added IDS that start with `templatetype_` + // this is only applicable if there are more than 1 request fields in protocol errValidate := validate.New().Struct(template) return out, multierr.Append(marshalErr, errValidate) } @@ -205,8 +290,9 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error if err != nil { return err } - // check if the template contains a multi protocols - if template.isMultiProtocol() { + // check if the template contains more than 1 protocol request + // if so preserve the order of the protocols and requests + if template.hasMultipleRequests() { var tempmap yaml.MapSlice err = unmarshal(&tempmap) if err != nil { @@ -221,37 +307,45 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error arr = append(arr, key) } // add protocols to the protocol stack (the idea is to preserve the order of the protocols) - template.addProtocolsToQueue(arr...) + template.addRequestsToQueue(arr...) } return nil } -// Internal function to create a protocol stack from a template if the template is a multi protocol template -func (template *Template) addProtocolsToQueue(keys ...string) { +// addProtocolsToQueue adds protocol requests to the queue and preserves order of the protocols and requests +func (template *Template) addRequestsToQueue(keys ...string) { for _, key := range keys { switch key { case types.DNSProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) case types.FileProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsFile)...) case types.HTTPProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) case types.HeadlessProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) case types.NetworkProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) case types.SSLProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) case types.WebsocketProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) case types.WHOISProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) + case types.CodeProtocol.String(): + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsCode)...) + // for deprecated protocols + case "requests": + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + case "network": + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) } } } -// isMultiProtocol checks if the template is a multi protocol template -func (template *Template) isMultiProtocol() bool { +// hasMultipleRequests checks if the template has multiple requests +// if so it preserves the order of the request during compile and execution +func (template *Template) hasMultipleRequests() bool { counter := len(template.RequestsDNS) + len(template.RequestsFile) + len(template.RequestsHTTP) + len(template.RequestsHeadless) + len(template.RequestsNetwork) + len(template.RequestsSSL) + @@ -279,8 +373,9 @@ func (template *Template) UnmarshalJSON(data []byte) error { if err != nil { return err } - // check if template contains multiple protocols - if template.isMultiProtocol() { + // check if the template contains more than 1 protocol request + // if so preserve the order of the protocols and requests + if template.hasMultipleRequests() { var tempMap map[string]interface{} err = json.Unmarshal(data, &tempMap) if err != nil { @@ -290,7 +385,7 @@ func (template *Template) UnmarshalJSON(data []byte) error { for k := range tempMap { arr = append(arr, k) } - template.addProtocolsToQueue(arr...) + template.addRequestsToQueue(arr...) } return nil } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index b4f392f062..8fb0315777 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -38,8 +38,6 @@ const ( WHOISProtocol // name:code CodeProtocol - // name: multi - MultiProtocol limit InvalidProtocol ) @@ -57,7 +55,6 @@ var protocolMappings = map[ProtocolType]string{ WebsocketProtocol: "websocket", WHOISProtocol: "whois", CodeProtocol: "code", - MultiProtocol: "multi", } func GetSupportedProtocolTypes() ProtocolTypes { diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 616e0c5ebf..893c9af08c 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -16,7 +16,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -91,8 +90,8 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco Browser: nil, Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - TemplateCtx: contextargs.New(), } + executerOpts.CreateTemplateCtxStore() return executerOpts } diff --git a/v2/pkg/tmplexec/README.md b/v2/pkg/tmplexec/README.md new file mode 100644 index 0000000000..26e80daf0a --- /dev/null +++ b/v2/pkg/tmplexec/README.md @@ -0,0 +1,11 @@ +# tmplexec + +tmplexec also known as template executer executes template it is different from `protocols` package which only contains logic within the scope of one protocol. tmplexec is resposible for executing `Template` with defined logic. with introduction of `multi protocol` and `flow` templates (deprecated package protocols/common/executer) did not seem appropriate/helpful anymore as it is outside of protocol scope and deals with execution of template which can contain 1 requests , or multiple requests of same protocol or multiple requests of different protocols. tmplexec is responsible for executing template and handling all logic related to it. + +## Engine/Backends + +Currently there are 3 engines for template execution + +- `Generic` => executes request[s] of same/one protocol +- `MultiProtocol` => executes requests of multiple protocols with shared logic between protocol requests see [multiprotocol](multiproto/README.md) +- `Flow` => executes requests of one or multiple protocol requests as specified by template in javascript (aka flow) [flow](flow/README.md) \ No newline at end of file diff --git a/v2/pkg/tmplexec/doc.go b/v2/pkg/tmplexec/doc.go new file mode 100644 index 0000000000..88207b9ec9 --- /dev/null +++ b/v2/pkg/tmplexec/doc.go @@ -0,0 +1,5 @@ +package tmplexec + +// tmplexec is package that provides +// template executors it is one level higher than protocols +// and deals with execution of entire template diff --git a/v2/pkg/tmplexec/exec.go b/v2/pkg/tmplexec/exec.go new file mode 100644 index 0000000000..aa2baf0bd5 --- /dev/null +++ b/v2/pkg/tmplexec/exec.go @@ -0,0 +1,165 @@ +package tmplexec + +import ( + "errors" + "fmt" + "strings" + "sync/atomic" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/multiproto" +) + +// TemplateExecutor is an executor for a template +type TemplateExecuter struct { + requests []protocols.Request + options *protocols.ExecutorOptions + engine TemplateEngine + results *atomic.Bool +} + +// Both executer & Executor are correct spellings (its open to interpretation) + +var _ protocols.Executer = &TemplateExecuter{} + +// NewTemplateExecuter creates a new request TemplateExecuter for list of requests +func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *TemplateExecuter { + isMultiProto := false + lastProto := "" + for _, request := range requests { + if request.Type().String() != lastProto && lastProto != "" { + isMultiProto = true + break + } + lastProto = request.Type().String() + } + + e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}} + if options.Flow != "" { + // we use a dummy input here because goal of flow executor at this point is to just check + // syntax and other things are correct before proceeding to actual execution + // during execution new instance of flow will be created as it is tightly coupled with lot of executor options + e.engine = flow.NewFlowExecutor(requests, contextargs.NewWithInput("dummy"), options, e.results) + } else { + // Review: + // multiproto engine is only used if there is more than one protocol in template + // else we use generic engine (should we use multiproto engine for single protocol with multiple requests as well ?) + if isMultiProto { + e.engine = multiproto.NewMultiProtocol(requests, options, e.results) + } else { + e.engine = generic.NewGenericEngine(requests, options, e.results) + } + } + + return e +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *TemplateExecuter) Compile() error { + cliOptions := e.options.Options + + for _, request := range e.requests { + if err := request.Compile(e.options); err != nil { + var dslCompilationError *dsl.CompilationError + if errors.As(err, &dslCompilationError) { + if cliOptions.Verbose { + rawErrorMessage := dslCompilationError.Error() + formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "." + gologger.Warning().Msgf(formattedErrorMessage) + gologger.Info().Msgf("The available custom DSL functions are:") + fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor)) + } + } + return err + } + } + return e.engine.Compile() +} + +// Requests returns the total number of requests the rule will perform +func (e *TemplateExecuter) Requests() int { + var count int + for _, request := range e.requests { + count += request.Requests() + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { + results := &atomic.Bool{} + defer func() { + // it is essential to remove template context of `Scan i.e template x input pair` + // since it is of no use after scan is completed (regardless of success or failure) + e.options.RemoveTemplateCtx(input.MetaInput) + }() + + var lastMatcherEvent *output.InternalWrappedEvent + writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { + if !results.Load() && matcherStatus { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + results.CompareAndSwap(false, true) + } + } + + cliExecutorCallback := func(event *output.InternalWrappedEvent) { + if event == nil { + // something went wrong + return + } + // If no results were found, and also interactsh is not being used + // in that case we can skip it, otherwise we've to show failure in + // case of matcher-status flag. + if !event.HasOperatorResult() && !event.UsesInteractsh { + lastMatcherEvent = event + } else { + if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + results.CompareAndSwap(false, true) + } else { + lastMatcherEvent = event + } + } + } + var err error + + // Note: this is required for flow executor + // flow executer is tightly coupled with lot of executor options + // and map , wg and other types earlier we tried to use (compile once and run multiple times) + // but it is causing lot of panic and nil pointer dereference issues + // so in compile step earlier we compile it to validate javascript syntax and other things + // and while executing we create new instance of flow executor everytime + if e.options.Flow != "" { + flowexec := flow.NewFlowExecutor(e.requests, input, e.options, results) + if err := flowexec.Compile(); err != nil { + return false, err + } + err = flowexec.ExecuteWithResults(input, cliExecutorCallback) + } else { + err = e.engine.ExecuteWithResults(input, cliExecutorCallback) + } + + if lastMatcherEvent != nil { + writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) + } + return results.Load(), err +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + gologger.Info().Msgf("[%s] Running on %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint()) + userCallback := func(event *output.InternalWrappedEvent) { + if event != nil { + callback(event) + } + } + return e.engine.ExecuteWithResults(input, userCallback) +} diff --git a/v2/pkg/tmplexec/flow/README.md b/v2/pkg/tmplexec/flow/README.md new file mode 100644 index 0000000000..0b4fc49814 --- /dev/null +++ b/v2/pkg/tmplexec/flow/README.md @@ -0,0 +1,319 @@ +# flow + +flow is a new template engine/backend introduced in v3 which primarily adds 2 most awaited features +- conditional execution of requests (ex: `flow: dns() && http()`) +- request execution orchestration (iterate over a slice, request execution order, if/for statement) + +both of these features are implemented using javascript (ECMAScript 5.1) with [goja](https://github.com/dop251/goja) backend. +## conditional execution + +Many times when writing complex templates we might need to add some extra checks (or conditional statements) before executing certain part of request + +An ideal example of this would be when bruteforcing wordpress login with default usernames and passwords. If we try to write a template for this it would be something like this +```yaml +id: wordpress-bruteforce +info: + name: WordPress Login Bruteforce + author: pdteam + severity: high + +http: + - method: POST + path: + - "{{BaseURL}}/wp-login.php" + payloads: + username: + - admin + - guest + - testuser + password: + - password123 + - qwertyuiop + - letmein + body: "log=§username§&pwd=§password§&wp-submit=Log+In" + attack: clusterbomb + matchers: + - type: word + words: + - "ERROR" + part: body + negative: true +``` + +but if we carefully re-evaluate this template, we can see that template is sending 9 requests without even checking, if the url actually exists or target site is actually a wordpress site. before v3 it was possible to do this by adding a extractor and sending additional content in say url fragment and it would fail if request was not successful and another way would be writing a workflow (2 templates and 1 workflow file i.e total 3 files for 1 template) but this is `hacky` and not a good solution. + +With addition of flow in Nuclei v3 we can re-write this template to first check if target is a wordpress site, if yes then bruteforce login with default credentials and this can be achieved by simply adding one line of content i.e `flow: http("check-wp") && http("bruteforce")` and nuclei will take care of everything else. + +```yaml +id: wordpress-bruteforce +info: + name: WordPress Login Bruteforce + author: pdteam + severity: high + +flow: http("check-wp") && http("bruteforce") + +http: + - id: check-wp + method: GET + path: + - "{{BaseURL}}/wp-login.php" + + matchers: + - type: word + words: + - "WordPress" + part: body + - type: word + words: + - "wp-content" + part: body + matchers-condition: and + + - id: bruteforce + method: POST + path: + - "{{BaseURL}}/wp-login.php" + payloads: + username: + - admin + - guest + - testuser + password: + - password123 + - qwertyuiop + - letmein + body: "log=§username§&pwd=§password§&wp-submit=Log+In" + attack: clusterbomb + matchers: + - type: word + words: + - "ERROR" + part: body + negative: true +``` +**Note:** this is just a example template with poor matchers. refer 'nuclei-templates' repo for final template + +The update template now seems straight forward and easy to understand. we are first checking if target is a wordpress site and then executing bruteforce requests. This is just a simple example of conditional execution and flow accepts any Javascript (ECMAScript 5.1) expression/code so you are free to craft any conditional execution logic you want using for,if and whatnot. + +## request execution orchestration + +`conditional execution` is one simple use case of flow but `flow` is much more powerful than that, for example it can be used to +- iterate over a slice of values and execute requests for each value (ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml)) +- extract values from one request and iterate over them and execute requests for each value ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml) +- get/set values from/to template context (global variables) +- print/log values to stdout at xyz condition or while debugging +- adding custom logic during template execution (ex: if status code is 403 => login and then re-run it) +- use any/all ECMAScript 5.1 javascript (like objects,arrays etc) and build/transform variables/input at runtime +- update variables at runtime (ex: when jwt expires update it by using refresh token and then continue execution) +- and a lot more (this is just a tip of iceberg) + +simply put request execution orchestration can be understood as nuclei logic bindings for javascript (i.e two way interaction between javascript and nuclei for a specific template) + +To better understand orchestration we can try to build a template for vhost enumeration using flow. which usually requires writing / using a new tool + +**for basic vhost enumeration a template should** +- do a PTR lookup for given ip +- get SSL ceritificate for given ip (i.e tls-grab) + - extract subject_cn from certificate + - extract subject_alt_names(SAN) from certificate + - filter out wildcard prefix from above values +- and finally bruteforce all found vhosts + + +**Now if we try to implement this in template it would be** +```yaml +# send a ssl request to get certificate +ssl: + - address: "{{Host}}:{{Port}}" + +# do a PTR lookup for given ip and get PTR value +dns: + - name: "{{FQDN}}" + type: PTR + + matchers: + - type: word + words: + - "IN\tPTR" + + extractors: + - type: regex + name: ptrValue + internal: true + group: 1 + regex: + - "IN\tPTR\t(.+)" + +# bruteforce all found vhosts +http: + - raw: + - | + GET / HTTP/1.1 + Host: {{vhost}} + + matchers: + - type: status + negative: true + status: + - 400 + - 502 + + extractors: + - type: dsl + dsl: + - '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length' tarun@macbook:~/Codebase/nuclei/integration_tes +``` +**But this template is not yet ready as it is missing core logic i.e how we use all these obtained data and do bruteforce** +and this is where flow comes into picture. flow is javascript code with two way bindings to nuclei. if we write javascript code to orchestrate vhost enumeration it is as simple as +```javascript + ssl(); + dns(); + for (let vhost of iterate(template["ssl_subject_cn"],template["ssl_subject_an"])) { + set("vhost", vhost); + http(); } +``` + +With just extra 5 lines of javascript code template can now perform vhost enumeration and this can be run on scale with all awesome features of nuclei with various supported inputs like ASN,CIDR,URL etc + + +In above Js code we are using some Nuclei JS bindings lets understand what they do + +- `ssl()` => execute ssl request +- `dns()` => execute dns request +- `template["ssl_subject_cn"]` => get value of `ssl_subject_cn` from template context (global variables) +- `iterate()` => this is a nuclei helper function which iterates any type of value (array,map,string,number) while handling empty / nil values +- `set("vhost",vhost)` => creates new variable `vhost` in template and assigns value of `vhost` to it +- `http()` => execute http request + + +This template is now missing one last thing i.e +- removing wildcard prefix (*.) in subject_cn,subject_an +- trailing `.` in PTR value + +and this can be done using either JS methods of using DSL helper functions as shown in below template + +```yaml +id: vhost-enum-flow + +info: + name: vhost enum flow + author: tarunKoyalwar + severity: info + description: | + vhost enumeration by extracting potential vhost names from ssl certificate and dns ptr records + +flow: | + ssl(); + dns({hide: true}); + for (let vhost of iterate(template["ssl_subject_cn"],template["ssl_subject_an"])) { + vhost = vhost.replace("*.", "") + set("vhost", vhost); + http(); + } + +ssl: + - address: "{{Host}}:{{Port}}" + +dns: + - name: "{{FQDN}}" + type: PTR + + matchers: + - type: word + words: + - "IN\tPTR" + + extractors: + - type: regex + name: ptrValue + internal: true + group: 1 + regex: + - "IN\tPTR\t(.+)" + +http: + - raw: + - | + GET / HTTP/1.1 + Host: {{trim_suffix(vhost, ".")}} + + matchers: + - type: status + negative: true + status: + - 400 + - 502 + + extractors: + - type: dsl + dsl: + - '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length' +``` + + +### Nuclei JS Bindings + +This section contains a brief description of all nuclei JS bindings and their usage + +**1. Protocol Execution Functions** + + Any protocol that is present in a nuclei template can be called/executed in javascript in format `proto_name()` i.e `http()` , `dns()` , `ssl()` etc + If we want to execute a specific request of a protocol (ref: see [nuclei-flow-dns](testcases/nuclei-flow-dns-id.yaml)) this can be achieved by either passing + - index of that request in protocol (ex: `dns(0)`, `dns(1)` etc) + - id of that request in protocol (ex: `dns("extract-vps")`, `dns("probe-http")` etc) + For More complex use cases multiple requests of a single protocol can be executed by just specifying their index or id one after another (ex: `dns("extract-vps","1")`) + +**2. Iterate Helper Function** + + Iterate is a nuclei js helper function which can be used to iterate over any type of value (array,map,string,number) while handling empty / nil values. + This is addon helper function from nuclei to omit boilerplate code of checking if value is empty or not and then iterating over it + ```javascript + iterate(123,{"a":1,"b":2,"c":3}) + // iterate over array with custom separator + iterate([1,2,3,4,5], " ") + ``` + **Note:** In above example we used `iterate(template["ssl_subject_cn"],template["ssl_subject_an"])` which removed lot of boilerplate code of checking if value is empty or not and then iterating over it + +**3. Set Helper Function** + + When Iterating over a values/array or some other use case we might want to invoke a request with custom/given value and this can be achieved by using `set()` helper function. When invoked/called it adds given variable to template context (global variables) and that value is used during execution of request/protocol. the format of `set()` is `set("variable_name",value)` ex: `set("username","admin")` etc + ```javascript + for (let vhost of myArray) { + set("vhost", vhost); + http(1) + } + ``` + **Note:** In above example we used `set("vhost", vhost)` which added `vhost` to template context (global variables) and then called `http(1)` which used this value in request + +**4. Template Context** + + when using `nuclei -jsonl` flag we get lot of data/metadata related to a vulnerability (ex: template details,extracted-values and much more) . A template context is nothing but a map/JSON containing all this data along with internal/unexported data that is only available at runtime (ex: extracted values from previous requests, variables added using `set()` etc). This template context is available in javascript as `template` variable and can be used to access any data from it. ex: `template["ssl_subject_cn"]` , `template["ssl_subject_an"]` etc + ```javascript + template["ssl_subject_cn"] // returns value of ssl_subject_cn from template context which is available after executing ssl request + template["ptrValue"] // returns value of ptrValue which was extracted using regex with internal: true + ``` + Lot of times we don't known what all data is available in template context and this can be easily found by printing it to stdout using `log()` function + ```javascript + log(template) + ``` + +**5. Log Helper Function** + + It is a nuclei js alternative to `console.log` and this pretty prints map data in readable format + **Note:** This should be used for debugging purposed only as this prints data to stdout + +**6. Dedupe** + + Lot of times just having arrays/slices is not enough and we might need to remove duplicate variables . for example in earlier vhost enumeration we did not remove any duplicates as there is always a chance of duplicate values in `ssl_subject_cn` and `ssl_subject_an` and this can be achieved by using `dedupe()` object. This is nuclei js helper function to abstract away boilerplate code of removing duplicates from array/slice + ```javascript + let uniq = new Dedupe(); // create new dedupe object + uniq.Add(template["ptrValue"]) + uniq.Add(template["ssl_subject_cn"]); + uniq.Add(template["ssl_subject_an"]); + log(uniq.Values()) + ``` + And that's it , this automatically converts any slice/array to map and removes duplicates from it and returns a slice/array of unique values + +------ +> Similar to DSL helper functions . we can either use built in functions available with `Javscript (ECMAScript 5.1)` or use DSL helper functions and its upto user to decide which one to uses \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/builtin/dedupe.go b/v2/pkg/tmplexec/flow/builtin/dedupe.go new file mode 100644 index 0000000000..eae088db8a --- /dev/null +++ b/v2/pkg/tmplexec/flow/builtin/dedupe.go @@ -0,0 +1,67 @@ +package builtin + +import ( + "crypto/md5" + "reflect" + + "github.com/dop251/goja" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Dedupe is a javascript builtin for deduping values +type Dedupe struct { + m map[string]goja.Value + VM *goja.Runtime +} + +// Add adds a value to the dedupe +func (d *Dedupe) Add(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) + } + } + for _, v := range allVars { + hash := hashValue(v) + if _, ok := d.m[hash]; ok { + continue + } + d.m[hash] = d.VM.ToValue(v) + } + return d.VM.ToValue(true) +} + +// Values returns all values from the dedupe +func (d *Dedupe) Values(call goja.FunctionCall) goja.Value { + tmp := []goja.Value{} + for _, v := range d.m { + tmp = append(tmp, v) + } + return d.VM.ToValue(tmp) +} + +// NewDedupe creates a new dedupe builtin object +func NewDedupe(vm *goja.Runtime) *Dedupe { + return &Dedupe{ + m: make(map[string]goja.Value), + VM: vm, + } +} + +// hashValue returns a hash of the value +func hashValue(value interface{}) string { + res := types.ToString(value) + md5sum := md5.Sum([]byte(res)) + return string(md5sum[:]) +} diff --git a/v2/pkg/tmplexec/flow/doc.go b/v2/pkg/tmplexec/flow/doc.go new file mode 100644 index 0000000000..42882657be --- /dev/null +++ b/v2/pkg/tmplexec/flow/doc.go @@ -0,0 +1 @@ +package flow diff --git a/v2/pkg/tmplexec/flow/flow_executor.go b/v2/pkg/tmplexec/flow/flow_executor.go new file mode 100644 index 0000000000..06abdc3243 --- /dev/null +++ b/v2/pkg/tmplexec/flow/flow_executor.go @@ -0,0 +1,240 @@ +package flow + +import ( + "fmt" + "io" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/dop251/goja" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + mapsutil "github.com/projectdiscovery/utils/maps" + "go.uber.org/multierr" +) + +var ( + // ErrInvalidRequestID is a request id error + ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided") +) + +// FlowExecutor is a flow executor for executing a flow +type FlowExecutor struct { + input *contextargs.Context + options *protocols.ExecutorOptions + + // javascript runtime reference and compiled program + jsVM *goja.Runtime + program *goja.Program // compiled js program + + // protocol requests and their callback functions + allProtocols map[string][]protocols.Request + protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js + callback func(event *output.InternalWrappedEvent) // result event callback + + // logic related variables + wg sync.WaitGroup + results *atomic.Bool + allErrs mapsutil.SyncLockMap[string, error] +} + +// NewFlowExecutor creates a new flow executor from a list of requests +func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor { + allprotos := make(map[string][]protocols.Request) + for _, req := range requests { + switch req.Type() { + case templateTypes.DNSProtocol: + allprotos[templateTypes.DNSProtocol.String()] = append(allprotos[templateTypes.DNSProtocol.String()], req) + case templateTypes.HTTPProtocol: + allprotos[templateTypes.HTTPProtocol.String()] = append(allprotos[templateTypes.HTTPProtocol.String()], req) + case templateTypes.NetworkProtocol: + allprotos[templateTypes.NetworkProtocol.String()] = append(allprotos[templateTypes.NetworkProtocol.String()], req) + case templateTypes.FileProtocol: + allprotos[templateTypes.FileProtocol.String()] = append(allprotos[templateTypes.FileProtocol.String()], req) + case templateTypes.HeadlessProtocol: + allprotos[templateTypes.HeadlessProtocol.String()] = append(allprotos[templateTypes.HeadlessProtocol.String()], req) + case templateTypes.SSLProtocol: + allprotos[templateTypes.SSLProtocol.String()] = append(allprotos[templateTypes.SSLProtocol.String()], req) + case templateTypes.WebsocketProtocol: + allprotos[templateTypes.WebsocketProtocol.String()] = append(allprotos[templateTypes.WebsocketProtocol.String()], req) + case templateTypes.WHOISProtocol: + allprotos[templateTypes.WHOISProtocol.String()] = append(allprotos[templateTypes.WHOISProtocol.String()], req) + case templateTypes.CodeProtocol: + allprotos[templateTypes.CodeProtocol.String()] = append(allprotos[templateTypes.CodeProtocol.String()], req) + default: + gologger.Error().Msgf("invalid request type %s", req.Type().String()) + } + } + f := &FlowExecutor{ + allProtocols: allprotos, + options: options, + allErrs: mapsutil.SyncLockMap[string, error]{ + ReadOnly: atomic.Bool{}, + Map: make(map[string]error), + }, + protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, + results: results, + jsVM: goja.New(), + input: input, + } + return f +} + +// Compile compiles js program and registers all functions +func (f *FlowExecutor) Compile() error { + if f.results == nil { + f.results = new(atomic.Bool) + } + // load all variables and evaluate with existing data + variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) + // cli options + optionVars := generators.BuildPayloadFromOptions(f.options.Options) + // constants + constants := f.options.Constants + allVars := generators.MergeMaps(variableMap, constants, optionVars) + // we support loading variables from files in variables , cli options and constants + // try to load if files exist + for k, v := range allVars { + if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) { + if value, err := f.ReadDataFromFile(str); err == nil { + allVars[k] = value + } else { + gologger.Warning().Msgf("could not load file '%s' for variable '%s': %s", str, k, err) + } + } + } + f.options.GetTemplateCtx(f.input.MetaInput).Merge(allVars) // merge all variables into template context + + // ---- define callback functions/objects---- + f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} + // iterate over all protocols and generate callback functions for each protocol + for p, requests := range f.allProtocols { + // for each protocol build a requestMap with reqID and protocol request + reqMap := mapsutil.Map[string, protocols.Request]{} + counter := 0 + proto := strings.ToLower(p) // donot use loop variables in callback functions directly + for index := range requests { + request := f.allProtocols[proto][index] + if request.GetID() != "" { + // if id is present use it + reqMap[request.GetID()] = request + } + // fallback to using index as id + // always allow index as id as a fallback + reqMap[strconv.Itoa(counter)] = request + counter++ + } + // ---define hook that allows protocol/request execution from js----- + // --- this is the actual callback that is executed when function is invoked in js---- + f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { + opts := &ProtoOptions{ + protoName: proto, + } + for _, v := range call.Arguments { + switch value := v.Export().(type) { + case map[string]interface{}: + opts.LoadOptions(value) + default: + opts.reqIDS = append(opts.reqIDS, types.ToString(value)) + } + } + // parallel execution of protocols + if opts.Async { + f.wg.Add(1) + go func() { + defer f.wg.Done() + f.requestExecutor(reqMap, opts) + }() + return f.jsVM.ToValue(true) + } + + return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) + } + } + return f.registerBuiltInFunctions() +} + +// ExecuteWithResults executes the flow and returns results +func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + defer func() { + if e := recover(); e != nil { + gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", input.MetaInput.Input, e) + panic(e) + } + }() + + f.callback = callback + f.input = input + // -----Load all types of variables----- + // add all input args to template context + if f.input != nil && f.input.HasArgs() { + f.input.ForEach(func(key string, value interface{}) { + f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value) + }) + } + if f.callback == nil { + return fmt.Errorf("output callback cannot be nil") + } + // pass flow and execute the js vm and handle errors + value, err := f.jsVM.RunProgram(f.program) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) + } + f.wg.Wait() + runtimeErr := f.GetRuntimeErrors() + if runtimeErr != nil { + return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") + } + if value.Export() != nil { + f.results.Store(value.ToBoolean()) + } else { + f.results.Store(true) + } + return nil +} + +// GetRuntimeErrors returns all runtime errors (i.e errors from all protocol combined) +func (f *FlowExecutor) GetRuntimeErrors() error { + errs := []error{} + for proto, err := range f.allErrs.GetAll() { + errs = append(errs, errorutil.NewWithErr(err).Msgf("failed to execute %v protocol", proto)) + } + return multierr.Combine(errs...) +} + +// ReadDataFromFile reads data from file respecting sandbox options +func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { + values := []string{} + // load file respecting sandbox + reader, err := f.options.Options.LoadHelperFile(payload, f.options.TemplatePath, f.options.Catalog) + if err != nil { + return values, err + } + defer reader.Close() + bin, err := io.ReadAll(reader) + if err != nil { + return values, err + } + for _, line := range strings.Split(string(bin), "\n") { + line = strings.TrimSpace(line) + if line != "" { + values = append(values, line) + } + } + return values, nil +} + +// Name returns the type of engine +func (f *FlowExecutor) Name() string { + return "flow" +} diff --git a/v2/pkg/tmplexec/flow/flow_executor_test.go b/v2/pkg/tmplexec/flow/flow_executor_test.go new file mode 100644 index 0000000000..7c7ec983c5 --- /dev/null +++ b/v2/pkg/tmplexec/flow/flow_executor_test.go @@ -0,0 +1,173 @@ +package flow_test + +import ( + "context" + "log" + "testing" + "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/parsers" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/ratelimit" + "github.com/stretchr/testify/require" +) + +var executerOpts protocols.ExecutorOptions + +func setup() { + options := testutils.DefaultOptions + testutils.Init(options) + progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + + executerOpts = protocols.ExecutorOptions{ + Output: testutils.NewMockOutputWriter(), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + } + workflowLoader, err := parsers.NewLoader(&executerOpts) + if err != nil { + log.Fatalf("Could not create workflow loader: %s\n", err) + } + executerOpts.WorkflowLoader = workflowLoader +} + +func TestFlowTemplateWithIndex(t *testing.T) { + // test + setup() + Template, err := templates.Parse("testcases/nuclei-flow-dns.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + input := contextargs.NewWithInput("hackerone.com") + gotresults, err := Template.Executer.Execute(input) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) +} + +func TestFlowTemplateWithID(t *testing.T) { + setup() + // apart from parse->compile->execution this testcase checks support for use custom id for protocol request and invocation of + // the same in js + Template, err := templates.Parse("testcases/nuclei-flow-dns-id.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + target := contextargs.NewWithInput("hackerone.com") + gotresults, err := Template.Executer.Execute(target) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) +} + +func TestFlowWithProtoPrefix(t *testing.T) { + // test + setup() + + // apart from parse->compile->execution this testcase checks + // mix of custom protocol request id and index is supported in js + // and also validates availability of protocol response variables in template context + Template, err := templates.Parse("testcases/nuclei-flow-dns-prefix.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + input := contextargs.NewWithInput("hackerone.com") + gotresults, err := Template.Executer.Execute(input) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) +} + +func TestFlowWithConditionNegative(t *testing.T) { + setup() + + // apart from parse->compile->execution this testcase checks + // if bitwise operator (&&) are properly executed and working + Template, err := templates.Parse("testcases/condition-flow.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + input := contextargs.NewWithInput("scanme.sh") + // expect no results and verify thant dns request is executed and http is not + gotresults, err := Template.Executer.Execute(input) + require.Nil(t, err, "could not execute template") + require.False(t, gotresults) +} + +func TestFlowWithConditionPositive(t *testing.T) { + setup() + + // apart from parse->compile->execution this testcase checks + // if bitwise operator (&&) are properly executed and working + Template, err := templates.Parse("testcases/condition-flow.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + input := contextargs.NewWithInput("blog.projectdiscovery.io") + // positive match . expect results also verify that both dns() and http() were executed + gotresults, err := Template.Executer.Execute(input) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) +} + +func TestFlowWithNoMatchers(t *testing.T) { + // when using conditional flow with no matchers at all + // we implicitly assume that request was successful and internally changed the result to true (for scope of condition only) + + // testcase-1 : no matchers but contains extractor + Template, err := templates.Parse("testcases/condition-flow-extractors.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + // positive match . expect results also verify that both dns() and http() were executed + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + + // testcase-2 : no matchers and no extractors + Template, err = templates.Parse("testcases/condition-flow-no-operators.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + // positive match . expect results also verify that both dns() and http() were executed + gotresults, err = Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + +} diff --git a/v2/pkg/tmplexec/flow/flow_internal.go b/v2/pkg/tmplexec/flow/flow_internal.go new file mode 100644 index 0000000000..15765a1476 --- /dev/null +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -0,0 +1,217 @@ +package flow + +import ( + "reflect" + "sync/atomic" + + "github.com/dop251/goja" + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow/builtin" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + mapsutil "github.com/projectdiscovery/utils/maps" +) + +// contains all internal/unexported methods of flow + +// requestExecutor executes a protocol/request and returns true if any matcher was found +func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { + defer func() { + // evaluate all variables after execution of each protocol + variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) + f.options.GetTemplateCtx(f.input.MetaInput).Merge(variableMap) // merge all variables into template context + + // to avoid polling update template variables everytime we execute a protocol + var m map[string]interface{} = f.options.GetTemplateCtx(f.input.MetaInput).GetAll() + _ = f.jsVM.Set("template", m) + }() + matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool + // if no id is passed execute all requests in sequence + if len(opts.reqIDS) == 0 { + // execution logic for http()/dns() etc + for index := range f.allProtocols[opts.protoName] { + req := f.allProtocols[opts.protoName][index] + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + if !opts.Hide { + f.callback(result) + } + // export dynamic values from operators (i.e internal:true) + // add add it to template context + // this is a conflicting behaviour with iterate-all + if result.HasOperatorResult() { + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) + } + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) + } + } + } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) + } + } + }) + if err != nil { + // save all errors in a map with id as key + // its less likely that there will be race condition but just in case + id := req.GetID() + if id == "" { + id, _ = reqMap.GetKeyWithValue(req) + } + err = f.allErrs.Set(opts.protoName+":"+id, err) + if err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } + return matcherStatus.Load() + } + } + return matcherStatus.Load() + } + + // execution logic for http("0") or http("get-aws-vpcs") + for _, id := range opts.reqIDS { + req, ok := reqMap[id] + if !ok { + gologger.Error().Msgf("invalid request id '%s' provided", id) + // compile error + if err := f.allErrs.Set(opts.protoName+":"+id, ErrInvalidRequestID.Msgf(id)); err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } + return matcherStatus.Load() + } + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + if !opts.Hide { + f.callback(result) + } + // export dynamic values from operators (i.e internal:true) + // add add it to template context + if result.HasOperatorResult() { + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) + } + _ = f.jsVM.Set("template", f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) + } + } + } + }) + if err != nil { + index := id + err = f.allErrs.Set(opts.protoName+":"+index, err) + if err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } + } + } + return matcherStatus.Load() +} + +// registerBuiltInFunctions registers all built in functions for the flow +func (f *FlowExecutor) registerBuiltInFunctions() error { + // currently we register following builtin functions + // log -> log to stdout with [JS] prefix should only be used for debugging + // set -> set a variable in template context + // proto(arg ...String) <- this is generic syntax of how a protocol/request binding looks in js + // we only register only those protocols that are available in template + + // we also register a map datatype called template with all template variables + // template -> all template variables are available in js template object + + if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { + // TODO: verify string interpolation and handle multiple args + arg := call.Argument(0).Export() + switch value := arg.(type) { + case string: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + case map[string]interface{}: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) + default: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + } + return goja.Null() + }); err != nil { + return err + } + + if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { + varName := call.Argument(0).Export() + varValue := call.Argument(1).Export() + f.options.GetTemplateCtx(f.input.MetaInput).Set(types.ToString(varName), varValue) + return goja.Null() + }); err != nil { + return err + } + + // iterate provides global iterator function by handling null values or strings + if err := f.jsVM.Set("iterate", func(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) + } + } + return f.jsVM.ToValue(allVars) + }); err != nil { + return err + } + + // add a builtin dedupe object + if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { + d := builtin.NewDedupe(f.jsVM) + obj := call.This + // register these methods + _ = obj.Set("Add", d.Add) + _ = obj.Set("Values", d.Values) + return nil + }); err != nil { + return err + } + + var m = f.options.GetTemplateCtx(f.input.MetaInput).GetAll() + if m == nil { + m = map[string]interface{}{} + } + + if err := f.jsVM.Set("template", m); err != nil { + // all template variables are available in js template object + return err + } + + // register all protocols + for name, fn := range f.protoFunctions { + if err := f.jsVM.Set(name, fn); err != nil { + return err + } + } + + program, err := goja.Compile("flow", f.options.Flow, false) + if err != nil { + return err + } + f.program = program + return nil +} diff --git a/v2/pkg/tmplexec/flow/options.go b/v2/pkg/tmplexec/flow/options.go new file mode 100644 index 0000000000..3d845a13b3 --- /dev/null +++ b/v2/pkg/tmplexec/flow/options.go @@ -0,0 +1,48 @@ +package flow + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// ProtoOptions are options that can be passed to flow protocol callback +// ex: dns(protoOptions) <- protoOptions are optional and can be anything +type ProtoOptions struct { + Hide bool + Async bool + protoName string + reqIDS []string +} + +// Examples +// dns() <- callback without any options +// dns(1) or dns(1,3) <- callback with index of protocol in template request at 1 or 1 and 3 +// dns("probe-http") or dns("extract-vpc","probe-http") <- callback with id's instead of index of request in template +// dns({hide:true}) or dns({hide:true,async:true}) <- callback with protocol options +// hide - hides result/event from output & sdk +// async - executes protocols in parallel (implicit wait no need to specify wait) +// Note: all of these options are optional and can be combined together in any order + +// LoadOptions loads the protocol options from a map +func (P *ProtoOptions) LoadOptions(m map[string]interface{}) { + P.Hide = GetBool(m["hide"]) + P.Async = GetBool(m["async"]) +} + +// GetBool returns bool value from interface +func GetBool(value interface{}) bool { + if value == nil { + return false + } + switch v := value.(type) { + case bool: + return v + default: + tmpValue := types.ToString(value) + if strings.EqualFold(tmpValue, "true") { + return true + } + } + return false +} diff --git a/v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml b/v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml new file mode 100644 index 0000000000..8dcb7c4f06 --- /dev/null +++ b/v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml @@ -0,0 +1,29 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + extractors: + - type: dsl + name: cname + internal: true + dsl: + - cname + +http: + - method: GET + path: + - "{{BaseURL}}?ref={{cname}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml b/v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml new file mode 100644 index 0000000000..8cb687b248 --- /dev/null +++ b/v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml @@ -0,0 +1,23 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + + +dns: + - name: "{{FQDN}}" + type: CNAME + +http: + - method: GET + path: + - "{{BaseURL}}?ref={{dns_cname}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/testcases/condition-flow.yaml b/v2/pkg/tmplexec/flow/testcases/condition-flow.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/v2/pkg/tmplexec/flow/testcases/condition-flow.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml new file mode 100644 index 0000000000..d13121fa0b --- /dev/null +++ b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml @@ -0,0 +1,42 @@ +id: nuclei-flow-dns + +info: + name: Nuclei flow dns + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("fetch-ns"); + template["nameservers"].forEach(nameserver => { + set("nameserver",nameserver); + dns("probe-ns"); + }); + +dns: + - id: "fetch-ns" + name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - id: "probe-ns" + name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml new file mode 100644 index 0000000000..6c040ff7b7 --- /dev/null +++ b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml @@ -0,0 +1,41 @@ +id: nuclei-flow-dns + +info: + name: Nuclei flow dns + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("0"); + template["nameservers"].forEach(nameserver => { + set("nameserver",nameserver); + dns("probe-ns"); + }); + +dns: + - name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - id: "probe-ns" + name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml new file mode 100644 index 0000000000..5be54ef27d --- /dev/null +++ b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml @@ -0,0 +1,40 @@ +id: nuclei-flow-dns + +info: + name: Nuclei flow dns + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("0"); + template["nameservers"].forEach(nameserver => { + set("nameserver",nameserver); + dns("1"); + }); + +dns: + - name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/util.go b/v2/pkg/tmplexec/flow/util.go new file mode 100644 index 0000000000..6e1c343f88 --- /dev/null +++ b/v2/pkg/tmplexec/flow/util.go @@ -0,0 +1,23 @@ +package flow + +import "github.com/projectdiscovery/nuclei/v2/pkg/operators" + +// Checks if template has matchers +func hasMatchers(all []*operators.Operators) bool { + for _, operator := range all { + if len(operator.Matchers) > 0 { + return true + } + } + return false +} + +// hasOperators checks if template has operators (i.e matchers/extractors) +func hasOperators(all []*operators.Operators) bool { + for _, operator := range all { + if operator != nil { + return true + } + } + return false +} diff --git a/v2/pkg/tmplexec/generic/exec.go b/v2/pkg/tmplexec/generic/exec.go new file mode 100644 index 0000000000..c4c0af7509 --- /dev/null +++ b/v2/pkg/tmplexec/generic/exec.go @@ -0,0 +1,94 @@ +package generic + +import ( + "strings" + "sync/atomic" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +) + +// generic engine as name suggests is a generic template +// execution engine and executes all requests one after another +// without any logic in between +type Generic struct { + requests []protocols.Request + options *protocols.ExecutorOptions + results *atomic.Bool +} + +// NewGenericEngine creates a new generic engine from a list of requests +func NewGenericEngine(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *Generic { + if results == nil { + results = &atomic.Bool{} + } + return &Generic{requests: requests, options: options, results: results} +} + +// Compile engine specific compilation +func (g *Generic) Compile() error { + // protocol/ request is already handled by template executer + return nil +} + +// ExecuteWithResults executes the template and returns results +func (g *Generic) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + dynamicValues := make(map[string]interface{}) + if input.HasArgs() { + input.ForEach(func(key string, value interface{}) { + dynamicValues[key] = value + }) + } + previous := make(map[string]interface{}) + + for _, req := range g.requests { + inputItem := input.Clone() + if g.options.InputHelper != nil && input.MetaInput.Input != "" { + if inputItem.MetaInput.Input = g.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" { + return nil + } + } + + err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + if event == nil { + // ideally this should never happen since protocol exits on error and callback is not called + return + } + ID := req.GetID() + if ID != "" { + builder := &strings.Builder{} + for k, v := range event.InternalEvent { + builder.WriteString(ID) + builder.WriteString("_") + builder.WriteString(k) + previous[builder.String()] = v + builder.Reset() + } + } + if event.HasOperatorResult() { + g.results.CompareAndSwap(false, true) + } + // for ExecuteWithResults : this callback will execute user defined callback and some error handling + // for Execute : this callback will print the result to output + callback(event) + }) + if err != nil { + if g.options.HostErrorsCache != nil { + g.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) + } + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, input.MetaInput.PrettyPrint(), err) + } + // If a match was found and stop at first match is set, break out of the loop and return + if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) { + break + } + } + return nil +} + +// Type returns the type of engine +func (g *Generic) Name() string { + return "generic" +} diff --git a/v2/pkg/tmplexec/interface.go b/v2/pkg/tmplexec/interface.go new file mode 100644 index 0000000000..ec52f915e0 --- /dev/null +++ b/v2/pkg/tmplexec/interface.go @@ -0,0 +1,33 @@ +package tmplexec + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/multiproto" +) + +var ( + _ TemplateEngine = &generic.Generic{} + _ TemplateEngine = &flow.FlowExecutor{} + _ TemplateEngine = &multiproto.MultiProtocol{} +) + +// TemplateEngine is a template executor with different functionality +// Ex: +// 1. generic => executes all protocol requests one after another (Done) +// 2. flow => executes protocol requests based on how they are defined in flow (Done) +// 3. multiprotocol => executes multiple protocols in parallel (Done) +type TemplateEngine interface { + // Note: below methods only need to implement extra / engine specific functionality + // basic request compilation , callbacks , cli output callback etc are handled by `TemplateExecuter` and no need to do it again + // Extra Compilation (if any) + Compile() error + + // ExecuteWithResults executes the template and returns results + ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error + + // Name returns name of template engine + Name() string +} diff --git a/v2/pkg/protocols/multi/README.md b/v2/pkg/tmplexec/multiproto/README.md similarity index 68% rename from v2/pkg/protocols/multi/README.md rename to v2/pkg/tmplexec/multiproto/README.md index f7c665cdb8..7f874a8aed 100644 --- a/v2/pkg/protocols/multi/README.md +++ b/v2/pkg/tmplexec/multiproto/README.md @@ -1,9 +1,8 @@ ## multi protocol execution ### Implementation -when template is unmarshalled, if it uses more than one protocol, it will be converted to a multi protocol -and the order of the protocols will be preserved as they were in the template and are stored in Request.Queue -when template is compiled , we iterate over queue and compile all the requests in the queue +when template is unmarshalled, if it uses more than one protocol, then order of protocols is preserved and is same is passed to Executor +multiproto is engine/backend for TemplateExecutor which takes care of sharing logic between protocols and executing them in order ### Execution when multi protocol template is executed , all protocol requests present in Queue are executed in order @@ -13,21 +12,14 @@ and dynamic values extracted are added to template context. apart from extracted `internal:true` values response fields/values of protocol are added to template context at `ExecutorOptions.TemplateCtx` which takes care of sync and other issues if any. all response fields are prefixed with template type prefix ex: `ssl_subject_dn` -### Other Methods -Such templates are usually used when a particular vulnerability requires more than one protocol to be executed -and in such cases the final result is core of the logic hence all methods such as -Ex: MakeResultEventItem, MakeResultEvent, GetCompiledOperators -are not implemented in multi protocol and just call the same method on last protocol in queue - - ### Adding New Protocol to multi protocol execution logic while logic/implementation of multi protocol execution is abstracted. it requires 3 statements to be added in newly implemented protocol to make response fields of that protocol available to global context -- Add `request.options.TemplateCtx.GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate` +- Add `request.options.GetTemplateCtx(f.input.MetaInput).GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate` ```go // example - values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll()) + values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(f.input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(values) ``` @@ -36,13 +28,13 @@ to make response fields of that protocol available to global context outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData) // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(),request.ID, outputEvent) ``` - Append all available template context values to outputEvent ```go // add variables from template context before matching/extraction - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(f.input.MetaInput).GetAll()) ``` adding these 3 statements takes care of all logic related to multi protocol execution diff --git a/v2/pkg/tmplexec/multiproto/doc.go b/v2/pkg/tmplexec/multiproto/doc.go new file mode 100644 index 0000000000..a22460da36 --- /dev/null +++ b/v2/pkg/tmplexec/multiproto/doc.go @@ -0,0 +1,4 @@ +package multiproto + +// multiproto is a template executer engine that executes multiple protocols +// with shared logic in between diff --git a/v2/pkg/tmplexec/multiproto/multi.go b/v2/pkg/tmplexec/multiproto/multi.go new file mode 100644 index 0000000000..9a4f37b68b --- /dev/null +++ b/v2/pkg/tmplexec/multiproto/multi.go @@ -0,0 +1,111 @@ +package multiproto + +import ( + "strconv" + "sync/atomic" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" +) + +// Mutliprotocol is a template executer engine that executes multiple protocols +// with logic in between +type MultiProtocol struct { + requests []protocols.Request + options *protocols.ExecutorOptions + results *atomic.Bool + readOnlyArgs map[string]interface{} // readOnlyArgs are readonly args that are available after compilation +} + +// NewMultiProtocol creates a new multiprotocol template engine from a list of requests +func NewMultiProtocol(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *MultiProtocol { + if results == nil { + results = &atomic.Bool{} + } + return &MultiProtocol{requests: requests, options: options, results: results} +} + +// Compile engine specific compilation +func (m *MultiProtocol) Compile() error { + // load all variables and evaluate with existing data + variableMap := m.options.Variables.GetAll() + // cli options + optionVars := generators.BuildPayloadFromOptions(m.options.Options) + // constants + constants := m.options.Constants + allVars := generators.MergeMaps(variableMap, constants, optionVars) + allVars = m.options.Variables.Evaluate(allVars) + m.readOnlyArgs = allVars + // no need to load files since they are done at template level + return nil +} + +// ExecuteWithResults executes the template and returns results +func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + // put all readonly args into template context + m.options.GetTemplateCtx(input.MetaInput).Merge(m.readOnlyArgs) + var finalProtoEvent *output.InternalWrappedEvent + // callback to process results from all protocols + multiProtoCallback := func(event *output.InternalWrappedEvent) { + if event != nil { + finalProtoEvent = event + } + // export dynamic values from operators (i.e internal:true) + if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 { + for k, v := range event.OperatorsResult.DynamicValues { + // TBD: iterate-all is only supported in `http` protocol + // we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context) + // currently if dynamic value array only contains one value we replace it with the value + if len(v) == 1 { + m.options.GetTemplateCtx(input.MetaInput).Set(k, v[0]) + } else { + // Note: if extracted value contains multiple values then they can be accessed by indexing + // ex: if values are dynamic = []string{"a","b","c"} then they are available as + // dynamic = "a" , dynamic1 = "b" , dynamic2 = "c" + // we intentionally omit first index for unknown situations (where no of extracted values are not known) + for i, val := range v { + if i == 0 { + m.options.GetTemplateCtx(input.MetaInput).Set(k, val) + } else { + m.options.GetTemplateCtx(input.MetaInput).Set(k+strconv.Itoa(i), val) + } + } + } + } + } + + // evaluate all variables after execution of each protocol + variableMap := m.options.Variables.Evaluate(m.options.GetTemplateCtx(input.MetaInput).GetAll()) + m.options.GetTemplateCtx(input.MetaInput).Merge(variableMap) // merge all variables into template context + } + + // template context: contains values extracted using `internal` extractor from previous protocols + // these values are extracted from each protocol in queue and are passed to next protocol in queue + // instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows) + // this makes it possible to use multi protocol templates in workflows + // Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow) + + // execute all protocols in the queue + for _, req := range m.requests { + values := m.options.GetTemplateCtx(input.MetaInput).GetAll() + err := req.ExecuteWithResults(input, output.InternalEvent(values), nil, multiProtoCallback) + // if error skip execution of next protocols + if err != nil { + return err + } + } + // Review: how to handle events of multiple protocols in a single template + // currently the outer callback is only executed once (for the last protocol in queue) + // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executem.go#L150 + // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables + callback(finalProtoEvent) + + return nil +} + +// Name of the template engine +func (m *MultiProtocol) Name() string { + return "multiproto" +} diff --git a/v2/pkg/protocols/multi/request_test.go b/v2/pkg/tmplexec/multiproto/multi_test.go similarity index 86% rename from v2/pkg/protocols/multi/request_test.go rename to v2/pkg/tmplexec/multiproto/multi_test.go index 660f69afe8..d09d02d739 100644 --- a/v2/pkg/protocols/multi/request_test.go +++ b/v2/pkg/tmplexec/multiproto/multi_test.go @@ -1,4 +1,4 @@ -package multi_test +package multiproto_test import ( "context" @@ -47,9 +47,9 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) { Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") - require.Equal(t, 2, len(Template.MultiProtoRequest.Queue)) + require.Equal(t, 2, len(Template.RequestsQueue)) - err = Template.MultiProtoRequest.Compile(&executerOpts) + err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) @@ -62,13 +62,11 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) { Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") - require.Equal(t, 3, len(Template.MultiProtoRequest.Queue)) + require.Equal(t, 3, len(Template.RequestsQueue)) - err = Template.MultiProtoRequest.Compile(&executerOpts) + err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - require.True(t, len(Template.MultiProtoRequest.GetCompiledOperators()) > 0, "could not compile operators") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) require.Nil(t, err, "could not execute template") require.True(t, gotresults) diff --git a/v2/pkg/protocols/multi/testcases/multiprotodynamic.yaml b/v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml similarity index 100% rename from v2/pkg/protocols/multi/testcases/multiprotodynamic.yaml rename to v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml diff --git a/v2/pkg/protocols/multi/testcases/multiprotowithprefix.yaml b/v2/pkg/tmplexec/multiproto/testcases/multiprotowithprefix.yaml similarity index 100% rename from v2/pkg/protocols/multi/testcases/multiprotowithprefix.yaml rename to v2/pkg/tmplexec/multiproto/testcases/multiprotowithprefix.yaml diff --git a/v2/pkg/types/interfaces.go b/v2/pkg/types/interfaces.go index 25dd685677..7b5bcd2ae8 100644 --- a/v2/pkg/types/interfaces.go +++ b/v2/pkg/types/interfaces.go @@ -77,6 +77,21 @@ func ToString(data interface{}) string { } } +// ToStringNSlice converts an interface to string in a quick way or to a slice with strings +// if the input is a slice of interfaces. +func ToStringNSlice(data interface{}) interface{} { + switch s := data.(type) { + case []interface{}: + var a []string + for _, v := range s { + a = append(a, ToString(v)) + } + return a + default: + return ToString(data) + } +} + func ToHexOrString(data interface{}) string { switch s := data.(type) { case string: diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 9aef497cf3..e6f88bc0eb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -2,12 +2,17 @@ package types import ( "io" + "os" + "path/filepath" "strings" "time" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -464,3 +469,29 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { } return optionalArguments } + +// LoadHelperFile loads a helper file needed for the template +// this respects the sandbox rules and only loads files from +// allowed directories +func (options *Options) LoadHelperFile(filePath, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { + if !options.AllowLocalFileAccess { + filePath = filepath.Clean(filePath) + templateAbsPath, err := filepath.Abs(templatePath) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not get absolute path") + } + templateDirectory := config.DefaultConfig.TemplatesDirectory + templatePathDir := filepath.Dir(templateAbsPath) + if !(templatePathDir != "/" && strings.HasPrefix(filePath, templatePathDir)) && !strings.HasPrefix(filePath, templateDirectory) { + return nil, errorutil.New("denied payload file path specified") + } + } + if catalog != nil { + return catalog.OpenFile(filePath) + } + f, err := os.Open(filePath) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not open file %v", filePath) + } + return f, nil +} diff --git a/v2/pkg/utils/insertion_ordered_map.go b/v2/pkg/utils/insertion_ordered_map.go index a9586778af..0e3b495d9b 100644 --- a/v2/pkg/utils/insertion_ordered_map.go +++ b/v2/pkg/utils/insertion_ordered_map.go @@ -58,7 +58,7 @@ func (insertionOrderedStringMap *InsertionOrderedStringMap) UnmarshalJSON(data [ } // toString converts an interface to string in a quick way -func toString(data interface{}) string { +func toString(data interface{}) interface{} { switch s := data.(type) { case nil: return "" @@ -92,6 +92,8 @@ func toString(data interface{}) string { return strconv.FormatUint(uint64(s), 10) case []byte: return string(s) + case []interface{}: + return data default: return fmt.Sprintf("%v", data) }