From f7fe99f80668918a8d6ca480418d7e54e784dda0 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Thu, 31 Aug 2023 18:03:01 +0530 Subject: [PATCH] add `flow` support in template (i.e javascript scripting) (#4015) * add flow logic * progress * working POC * fix string slice normalization issue in variables * update * fix nil panic * remove poll() * load file with sandbox and more * fix failing integration tests * JS: log: print in vardump format * fix missing id in protocols * fix proto prefix in template context * flow: add unit tests * conditional flow support using flow * fix proto callbacks + more unit tests * adds integration test * conditional flow: check if req has any matchers * fix lint error * deprecate iterate-all+ missing multi-proto implementation * fix ip input in raw request * JS: feat dedupe object+ more builtin funcs * feat: hide protocol result using hide * feat: async execution * complete async execution support * fix condition-flow without any matchers * refactor: template executer package (tmplexec) * flow executor working * fix data race in templateCtx * templateCtx redesign * fix failing unit test * add multiprotocol support to deprecated syntax * fix race condition in utils & tlsx * add documentation in flow package * remove regions.txt file * fix minor issue with self contained templates * fix typos of copilot * dep + misc update * fix reqID: use req.Type instead of template.Type --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .../flow/conditional-flow-negative.yaml | 27 ++ integration_tests/flow/conditional-flow.yaml | 27 ++ integration_tests/flow/dns-ns-probe.yaml | 42 +++ .../flow/iterate-values-flow.yaml | 35 ++ .../protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- .../protocols/code/py-interactsh.yaml | 2 +- .../protocols/code/py-snippet.yaml | 2 +- .../protocols/http/matcher-status.yaml | 2 +- v2/cmd/integration-test/flow.go | 84 +++++ v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/multi.go | 4 +- v2/cmd/integration-test/template-path.go | 2 +- v2/go.mod | 39 ++- v2/go.sum | 101 ++++-- v2/pkg/catalog/config/constants.go | 2 +- v2/pkg/protocols/code/code.go | 15 +- .../common/contextargs/contextargs.go | 32 +- .../protocols/common/contextargs/metainput.go | 19 ++ v2/pkg/protocols/common/executer/executer.go | 189 ----------- .../protocols/common/generators/generators.go | 9 +- .../common/generators/generators_test.go | 13 +- v2/pkg/protocols/common/generators/load.go | 26 +- .../protocols/common/generators/load_test.go | 101 ++++-- .../protocols/common/variables/variables.go | 19 ++ v2/pkg/protocols/dns/dns.go | 2 +- v2/pkg/protocols/dns/request.go | 12 +- v2/pkg/protocols/file/request.go | 18 +- .../headless/engine/page_actions_test.go | 4 +- v2/pkg/protocols/headless/headless.go | 5 +- v2/pkg/protocols/headless/request.go | 6 +- v2/pkg/protocols/http/build_request.go | 5 +- v2/pkg/protocols/http/http.go | 3 +- v2/pkg/protocols/http/raw/raw.go | 5 +- v2/pkg/protocols/http/request.go | 4 +- .../protocols/http/request_generator_test.go | 5 +- v2/pkg/protocols/multi/doc.go | 6 - v2/pkg/protocols/multi/request.go | 172 ---------- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/network/request.go | 14 +- v2/pkg/protocols/offlinehttp/request.go | 4 +- v2/pkg/protocols/protocols.go | 74 +++- v2/pkg/protocols/ssl/ssl.go | 13 +- v2/pkg/protocols/websocket/websocket.go | 17 +- v2/pkg/protocols/whois/whois.go | 9 +- v2/pkg/templates/compile.go | 82 +++-- v2/pkg/templates/compile_test.go | 14 - v2/pkg/templates/templates.go | 141 ++++++-- v2/pkg/templates/types/types.go | 3 - v2/pkg/testutils/testutils.go | 3 +- v2/pkg/tmplexec/README.md | 11 + v2/pkg/tmplexec/doc.go | 5 + v2/pkg/tmplexec/exec.go | 165 +++++++++ v2/pkg/tmplexec/flow/README.md | 319 ++++++++++++++++++ v2/pkg/tmplexec/flow/builtin/dedupe.go | 67 ++++ v2/pkg/tmplexec/flow/doc.go | 1 + v2/pkg/tmplexec/flow/flow_executor.go | 240 +++++++++++++ v2/pkg/tmplexec/flow/flow_executor_test.go | 173 ++++++++++ v2/pkg/tmplexec/flow/flow_internal.go | 217 ++++++++++++ v2/pkg/tmplexec/flow/options.go | 48 +++ .../testcases/condition-flow-extractors.yaml | 29 ++ .../condition-flow-no-operators.yaml | 23 ++ .../flow/testcases/condition-flow.yaml | 27 ++ .../flow/testcases/nuclei-flow-dns-id.yaml | 42 +++ .../testcases/nuclei-flow-dns-prefix.yaml | 41 +++ .../flow/testcases/nuclei-flow-dns.yaml | 40 +++ v2/pkg/tmplexec/flow/util.go | 23 ++ v2/pkg/tmplexec/generic/exec.go | 94 ++++++ v2/pkg/tmplexec/interface.go | 33 ++ .../multi => tmplexec/multiproto}/README.md | 20 +- v2/pkg/tmplexec/multiproto/doc.go | 4 + v2/pkg/tmplexec/multiproto/multi.go | 111 ++++++ .../multiproto/multi_test.go} | 12 +- .../testcases/multiprotodynamic.yaml | 0 .../testcases/multiprotowithprefix.yaml | 0 v2/pkg/types/interfaces.go | 15 + v2/pkg/types/types.go | 31 ++ v2/pkg/utils/insertion_ordered_map.go | 4 +- 78 files changed, 2576 insertions(+), 639 deletions(-) create mode 100644 integration_tests/flow/conditional-flow-negative.yaml create mode 100644 integration_tests/flow/conditional-flow.yaml create mode 100644 integration_tests/flow/dns-ns-probe.yaml create mode 100644 integration_tests/flow/iterate-values-flow.yaml create mode 100644 v2/cmd/integration-test/flow.go delete mode 100644 v2/pkg/protocols/common/executer/executer.go delete mode 100644 v2/pkg/protocols/multi/doc.go delete mode 100644 v2/pkg/protocols/multi/request.go create mode 100644 v2/pkg/tmplexec/README.md create mode 100644 v2/pkg/tmplexec/doc.go create mode 100644 v2/pkg/tmplexec/exec.go create mode 100644 v2/pkg/tmplexec/flow/README.md create mode 100644 v2/pkg/tmplexec/flow/builtin/dedupe.go create mode 100644 v2/pkg/tmplexec/flow/doc.go create mode 100644 v2/pkg/tmplexec/flow/flow_executor.go create mode 100644 v2/pkg/tmplexec/flow/flow_executor_test.go create mode 100644 v2/pkg/tmplexec/flow/flow_internal.go create mode 100644 v2/pkg/tmplexec/flow/options.go create mode 100644 v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml create mode 100644 v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml create mode 100644 v2/pkg/tmplexec/flow/testcases/condition-flow.yaml create mode 100644 v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml create mode 100644 v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml create mode 100644 v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml create mode 100644 v2/pkg/tmplexec/flow/util.go create mode 100644 v2/pkg/tmplexec/generic/exec.go create mode 100644 v2/pkg/tmplexec/interface.go rename v2/pkg/{protocols/multi => tmplexec/multiproto}/README.md (68%) create mode 100644 v2/pkg/tmplexec/multiproto/doc.go create mode 100644 v2/pkg/tmplexec/multiproto/multi.go rename v2/pkg/{protocols/multi/request_test.go => tmplexec/multiproto/multi_test.go} (86%) rename v2/pkg/{protocols/multi => tmplexec/multiproto}/testcases/multiprotodynamic.yaml (100%) rename v2/pkg/{protocols/multi => tmplexec/multiproto}/testcases/multiprotowithprefix.yaml (100%) diff --git a/integration_tests/flow/conditional-flow-negative.yaml b/integration_tests/flow/conditional-flow-negative.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/integration_tests/flow/conditional-flow-negative.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/integration_tests/flow/conditional-flow.yaml b/integration_tests/flow/conditional-flow.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/integration_tests/flow/conditional-flow.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/integration_tests/flow/dns-ns-probe.yaml b/integration_tests/flow/dns-ns-probe.yaml new file mode 100644 index 0000000000..569a9e766c --- /dev/null +++ b/integration_tests/flow/dns-ns-probe.yaml @@ -0,0 +1,42 @@ +id: dns-ns-probe + +info: + name: Nuclei flow dns ns probe + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("fetch-ns"); + for(let ns of template["nameservers"]) { + set("nameserver",ns); + dns("probe-ns"); + }; + +dns: + - id: "fetch-ns" + name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - id: "probe-ns" + name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/integration_tests/flow/iterate-values-flow.yaml b/integration_tests/flow/iterate-values-flow.yaml new file mode 100644 index 0000000000..3abea9e4cb --- /dev/null +++ b/integration_tests/flow/iterate-values-flow.yaml @@ -0,0 +1,35 @@ +id: extract-emails + +info: + name: Extract Email IDs from Response + author: pdteam + severity: info + + +flow: | + http(0) + for(let email of template["emails"]) { + set("email",email); + http(1); + } + +http: + - method: GET + path: + - "{{BaseURL}}" + + extractors: + - type: regex + name: emails + internal: true + regex: + - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' + + - method: GET + path: + - "{{BaseURL}}/user/{{base64(email)}}" + + matchers: + - type: word + words: + - "Welcome" \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 0e5c9c2881..b05ccf0923 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4b0a00483046022100cd2b9d34169cdb716caee25976fed763880435f2f1e2979c9d7c9d2bd7b8e409022100dd0ba8bd3fa6a6be5f964ca3b0ce8bdbb20d865553133cf494ef64fbeebff345 \ No newline at end of file +# digest: 4a0a00473045022100e17c7a809fd64419baf401b5331edab3a68a4c182f7777614beb1862eb6ea8b7022011b95fc0e22d7f82e08e01b56ce87afdbe03027c238ba290a058d695226173ae \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 8fae1b051a..ee76c50c32 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a00483046022100f663e5afaf5c118b21b9c5918cba12d7cc83edc2a3ee0f338c07e3cd1fe40e20022100b46193e3275c490a4ad3897c6e2ca51ce09f408538b17d041e0063d40f4df833 \ No newline at end of file +# digest: 490a004630440220241d7faae14ab5760dbe7acf621a3923d0650148bc14a52a9be06ba76e8e0abf02201885fcc432d354d3c99ea97b964838719451bc97f148229f187f61eee7525eb6 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index de4abdac08..664c2d7e8f 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4b0a00483046022100c45cd27b9d49879663e1ea3c877dc362d06b8a0aea64b1ab06be3af5aa9a32ee0221008f5ee347245a2c1e04c46528e4c70a5a851f95c6ba49d2834ef7c3784bca47a9 \ No newline at end of file +# digest: 490a004630440220427cb7100f0b7d95224f490a6f4f626186782cb26c69f2551d6aefcdbc7c17d20220206161ad3a98afe8fcef9dd06d9a6dd5f34c5f7e3cd3ab7f81328f033dcd2b48 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 9b0187dd8d..6b77ca5227 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022100df57bf446d6d8e73ff9424b1055faebcea9038e5d5934834ed8e619b77bdfd5e02201754c1cebe9f65883315b3830755a0689999f33db7102cd8d5469e4c01cc6a66 \ No newline at end of file +# digest: 4a0a00473045022056092462597e85139626656d37df123094cb3861bdf583450c38814bac8df9cb022100e83a8c552f8f8a098f6b7ec8a32c6b448b995e000884beadb50cb0f2720117de \ No newline at end of file diff --git a/integration_tests/protocols/http/matcher-status.yaml b/integration_tests/protocols/http/matcher-status.yaml index 5704c2a3d9..4cfd0d1a0b 100644 --- a/integration_tests/protocols/http/matcher-status.yaml +++ b/integration_tests/protocols/http/matcher-status.yaml @@ -1,4 +1,4 @@ -id: matchet-status +id: matcher-status info: name: Test Matcher Status diff --git a/v2/cmd/integration-test/flow.go b/v2/cmd/integration-test/flow.go new file mode 100644 index 0000000000..67d4b749b4 --- /dev/null +++ b/v2/cmd/integration-test/flow.go @@ -0,0 +1,84 @@ +package main + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var flowTestcases = []TestCaseInfo{ + {Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}}, + {Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, + {Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, + {Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, +} + +type conditionalFlow struct{} + +func (t *conditionalFlow) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug) + if err != nil { + return err + } + return expectResultsCount(results, 2) +} + +type conditionalFlowNegative struct{} + +func (t *conditionalFlowNegative) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + +type iterateValuesFlow struct{} + +func (t *iterateValuesFlow) Execute(filePath string) error { + router := httprouter.New() + testemails := []string{ + "secrets@scanme.sh", + "superadmin@scanme.sh", + } + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(fmt.Sprint(testemails))) + }) + router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) + }) + + router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) + }) + + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 2) +} + +type dnsNsProbe struct{} + +func (t *dnsNsProbe) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug) + if err != nil { + return err + } + return expectResultsCount(results, 3) +} + +func getBase64(input string) string { + return base64.StdEncoding.EncodeToString([]byte(input)) +} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 66d562ccff..f18560c773 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -49,6 +49,7 @@ var ( "multi": multiProtoTestcases, "generic": genericTestcases, "dsl": dslTestcases, + "flow": flowTestcases, } // For debug purposes diff --git a/v2/cmd/integration-test/multi.go b/v2/cmd/integration-test/multi.go index a7998ab358..a9ff58fffd 100644 --- a/v2/cmd/integration-test/multi.go +++ b/v2/cmd/integration-test/multi.go @@ -1,6 +1,8 @@ package main -import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) var multiProtoTestcases = []TestCaseInfo{ {Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}}, diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 773f919254..a4e483a405 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{ //template folder path issue {Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd - {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, + {Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, //relative path {Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path diff --git a/v2/go.mod b/v2/go.mod index 0539604078..4ced433be3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,17 +21,17 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.35 - github.com/projectdiscovery/hmap v0.0.13 + github.com/projectdiscovery/fastdialer v0.0.36 + github.com/projectdiscovery/hmap v0.0.15 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.18 - github.com/projectdiscovery/retryabledns v1.0.32 - github.com/projectdiscovery/retryablehttp-go v1.0.20 + github.com/projectdiscovery/retryabledns v1.0.35 + github.com/projectdiscovery/retryablehttp-go v1.0.24 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.23.6 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 @@ -39,9 +39,9 @@ require ( github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.12.0 - golang.org/x/oauth2 v0.10.0 - golang.org/x/text v0.11.0 + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 + golang.org/x/text v0.12.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) @@ -60,6 +60,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/docker/go-units v0.5.0 + github.com/dop251/goja v0.0.0-20230812105242-81d76064690d github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 github.com/h2non/filetype v1.1.3 @@ -68,7 +69,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.12 + github.com/projectdiscovery/goflags v0.1.18 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 github.com/projectdiscovery/httpx v1.3.4 @@ -76,9 +77,9 @@ require ( github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.1 + github.com/projectdiscovery/tlsx v1.1.4 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 + github.com/projectdiscovery/utils v0.0.51 github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -87,7 +88,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect @@ -105,14 +106,16 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect - github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaukas/godicttls v0.0.3 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.8 // indirect @@ -128,11 +131,13 @@ require ( github.com/projectdiscovery/asnmap v1.0.4 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect - github.com/refraction-networking/utls v1.3.2 // indirect + github.com/quic-go/quic-go v0.37.0 // indirect + github.com/refraction-networking/utls v1.4.2 // indirect github.com/sashabaranov/go-openai v1.14.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -216,10 +221,10 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 082cecc252..efaa4f96f8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -3,8 +3,8 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= @@ -128,6 +128,9 @@ github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM2 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -145,11 +148,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= -github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20230812105242-81d76064690d h1:9aaGwVf4q+kknu+mROAXUApJ1DoOwhE8dGj/XLBYzWg= +github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -179,6 +189,7 @@ github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw4 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -190,6 +201,9 @@ github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+j github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -234,6 +248,9 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -250,8 +267,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= @@ -259,6 +276,7 @@ github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDj github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU= @@ -295,6 +313,7 @@ github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -371,8 +390,9 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= @@ -398,20 +418,20 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= -github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= -github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= +github.com/projectdiscovery/fastdialer v0.0.36 h1:Ac/CRLryJB2mA8erDwAHoCJGFvjCDIPUznxWl9kJPW8= +github.com/projectdiscovery/fastdialer v0.0.36/go.mod h1:jxX9iQJdTwlD6u0Q9Dj9/AmatHPW2GRl3V6XTAvKtHY= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0= -github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= +github.com/projectdiscovery/goflags v0.1.18 h1:L4nwDBNJcZhbmhI3GhQ1GJwz7xVWFL3BumJ+TIDBi5E= +github.com/projectdiscovery/goflags v0.1.18/go.mod h1:cZut0Q98yksNVtM73RPSm22N/eDkAMFT9t6mwu6S5pY= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6/go.mod h1:jCpXNvLUCPMzm5AhJv8wtnUt/7rz0TY2SsqvKQ8tn2E= -github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= -github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= +github.com/projectdiscovery/hmap v0.0.15 h1:iTRXV94bNIuR5obYBxOhvs3yEYXdNdJJnrXnxv4uLHc= +github.com/projectdiscovery/hmap v0.0.15/go.mod h1:oybodscUwBbL4GnhBPPTemazPXyMErqL+dE+0ZtJ6lg= github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= @@ -426,31 +446,34 @@ github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt3 github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo= -github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c= -github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE= -github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI= +github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU= +github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c= +github.com/projectdiscovery/retryablehttp-go v1.0.24 h1:1In7vIUnNvEdHhnA5KmUVf+D+tVZgITaJUZxFYgKCdo= +github.com/projectdiscovery/retryablehttp-go v1.0.24/go.mod h1:S2KiViUrjvRua/mifKEj+6Gs8TJjhFsrZwOXKJAZzSA= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= -github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= +github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoWslEpIw= +github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 h1:bgTXdrA/yFhFGfjhMIsczVNhnsMEHFidgS/FD2Tq5Js= -github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/utils v0.0.51 h1:WZV8kP4VW/I1ZkXDjyoudhhfVVHpAKKnW+Re0LVNMbc= +github.com/projectdiscovery/utils v0.0.51/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= -github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= -github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= +github.com/quic-go/quic-go v0.37.0 h1:wf/Ym2yeWi98oQn4ahiBSqdnaXVxNQGj2oBQFgiVChc= +github.com/quic-go/quic-go v0.37.0/go.mod h1:XtCUOCALTTWbPyd0IxFfHf6h0sEMubRFvEYHl3QxKw8= +github.com/refraction-networking/utls v1.4.2 h1:7N+928mSM1pEyAJb8x2Y1FbEwTIftGwn2IFykosSzwc= +github.com/refraction-networking/utls v1.4.2/go.mod h1:JkUIj+Pc8eyFB0z+A4RJRZmoT43ajjFZWVMXuZQ8BEQ= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -463,8 +486,8 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= -github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -478,8 +501,9 @@ github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2Iqp github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= @@ -619,8 +643,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -653,12 +677,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -693,6 +717,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -706,9 +731,9 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -717,19 +742,20 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -767,6 +793,7 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nR gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 8b945c4ad6..2f6b4fbff8 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.0.0` + Version = `v3.0.0-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go index d601125466..f93af3fc93 100644 --- a/v2/pkg/protocols/code/code.go +++ b/v2/pkg/protocols/code/code.go @@ -32,6 +32,8 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` // description: | // Engine type Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine,enum=python,enum=powershell,enum=command"` @@ -96,7 +98,7 @@ func (request *Request) Requests() int { // GetID returns the ID for the request if any. func (request *Request) GetID() string { - return "" + return request.ID } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. @@ -115,10 +117,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // inject all template context values as gozero env variables variables := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) + // add template context values + variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) variablesMap := request.options.Variables.Evaluate(variables) - variables = generators.MergeMaps(variablesMap, variables, optionVars) + variables = generators.MergeMaps(variablesMap, variables, optionVars, request.options.Constants) for name, value := range variables { v := fmt.Sprint(value) v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs) @@ -151,6 +155,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo + // expose response variables in proto_var format + // this is no-op if the template is not a multi protocol template + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data) + + // add variables from template context before matching/extraction + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(interactshURLs, data) } diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index aa7dfa39b5..dd594f73d1 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -50,6 +50,36 @@ func (ctx *Context) hasArgs() bool { return !ctx.args.IsEmpty() } +// Merge the key-value pairs +func (ctx *Context) Merge(args map[string]interface{}) { + _ = ctx.args.Merge(args) +} + +// Add the specific key-value pair +func (ctx *Context) Add(key string, v interface{}) { + values, ok := ctx.args.Get(key) + if !ok { + ctx.Set(key, v) + } + + // If the key exists, append the value to the existing value + switch v := v.(type) { + case []string: + if values, ok := values.([]string); ok { + values = append(values, v...) + ctx.Set(key, values) + } + case string: + if values, ok := values.(string); ok { + tmp := []string{values, v} + ctx.Set(key, tmp) + } + default: + values, _ := ctx.Get(key) + ctx.Set(key, []interface{}{values, v}) + } +} + // Get the value with specific key if exists func (ctx *Context) Get(key string) (interface{}, bool) { if !ctx.hasArgs() { @@ -86,7 +116,7 @@ func (ctx *Context) HasArgs() bool { func (ctx *Context) Clone() *Context { newCtx := &Context{ MetaInput: ctx.MetaInput.Clone(), - args: ctx.args, + args: ctx.args.Clone(), CookieJar: ctx.CookieJar, } return newCtx diff --git a/v2/pkg/protocols/common/contextargs/metainput.go b/v2/pkg/protocols/common/contextargs/metainput.go index b76603de16..4027bd7d01 100644 --- a/v2/pkg/protocols/common/contextargs/metainput.go +++ b/v2/pkg/protocols/common/contextargs/metainput.go @@ -2,6 +2,7 @@ package contextargs import ( "bytes" + "crypto/md5" "fmt" "strings" @@ -14,6 +15,8 @@ type MetaInput struct { Input string `json:"input,omitempty"` // CustomIP to use for connection CustomIP string `json:"customIP,omitempty"` + // hash of the input + hash string `json:"-"` } func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) { @@ -67,3 +70,19 @@ func (metaInput *MetaInput) PrettyPrint() string { } return metaInput.Input } + +// GetScanHash returns a unique hash that represents a scan by hashing (metainput + templateId) +func (metaInput *MetaInput) GetScanHash(templateId string) string { + // there may be some cases where metainput is changed ex: while executing self-contained template etc + // but that totally changes the scanID/hash so to avoid that we compute hash only once + // and reuse it for all subsequent calls + if metaInput.hash == "" { + metaInput.hash = getMd5Hash(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP) + } + return metaInput.hash +} + +func getMd5Hash(data string) string { + bin := md5.Sum([]byte(data)) + return string(bin[:]) +} diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go deleted file mode 100644 index 4983ae1ed7..0000000000 --- a/v2/pkg/protocols/common/executer/executer.go +++ /dev/null @@ -1,189 +0,0 @@ -package executer - -import ( - "fmt" - "strings" - "sync/atomic" - - "github.com/pkg/errors" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" -) - -// Executer executes a group of requests for a protocol -type Executer struct { - requests []protocols.Request - options *protocols.ExecutorOptions -} - -var _ protocols.Executer = &Executer{} - -// NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *Executer { - return &Executer{requests: requests, options: options} -} - -// Compile compiles the execution generators preparing any requests possible. -func (e *Executer) Compile() error { - cliOptions := e.options.Options - - for _, request := range e.requests { - if err := request.Compile(e.options); err != nil { - var dslCompilationError *dsl.CompilationError - if errors.As(err, &dslCompilationError) { - if cliOptions.Verbose { - rawErrorMessage := dslCompilationError.Error() - formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "." - gologger.Warning().Msgf(formattedErrorMessage) - gologger.Info().Msgf("The available custom DSL functions are:") - fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor)) - } - } - return err - } - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int { - var count int - for _, request := range e.requests { - count += request.Requests() - } - return count -} - -// Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input *contextargs.Context) (bool, error) { - results := &atomic.Bool{} - - dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) { - dynamicValues[key] = value - }) - } - previous := make(map[string]interface{}) - - var lastMatcherEvent *output.InternalWrappedEvent - writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { - if !results.Load() && matcherStatus { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } - results.CompareAndSwap(false, true) - } - } - - for _, req := range e.requests { - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" { - return false, nil - } - } - - err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { - if event == nil { - // ideally this should never happen since protocol exits on error and callback is not called - return - } - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - previous[builder.String()] = v - builder.Reset() - } - } - // If no results were found, and also interactsh is not being used - // in that case we can skip it, otherwise we've to show failure in - // case of matcher-status flag. - if !event.HasOperatorResult() && !event.UsesInteractsh { - lastMatcherEvent = event - } else { - if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { - results.CompareAndSwap(false, true) - } else { - lastMatcherEvent = event - } - } - }) - if err != nil { - if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) - } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) - } - // If a match was found and stop at first match is set, break out of the loop and return - if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - break - } - } - if lastMatcherEvent != nil { - writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) - } - return results.Load(), nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { - dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) { - dynamicValues[key] = value - }) - } - previous := make(map[string]interface{}) - results := &atomic.Bool{} - - for _, req := range e.requests { - req := req - - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" { - return nil - } - } - - err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - previous[builder.String()] = v - builder.Reset() - } - } - if event.OperatorsResult == nil { - return - } - results.CompareAndSwap(false, true) - callback(event) - }) - if err != nil { - if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) - } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) - } - // If a match was found and stop at first match is set, break out of the loop and return - if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - break - } - } - return nil -} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 98fc6aa138..c17fef84c8 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // PayloadGenerator is the generator struct for generating payloads @@ -14,10 +14,11 @@ type PayloadGenerator struct { Type AttackType catalog catalog.Catalog payloads map[string][]string + options *types.Options } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, allowLocalFileAccess bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog, customAttackType string, opts *types.Options) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -38,12 +39,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } } - generator := &PayloadGenerator{catalog: catalog} + generator := &PayloadGenerator{catalog: catalog, options: opts} if err := generator.validate(payloadsFinal, templatePath); err != nil { return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, allowLocalFileAccess) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 7187808904..2226188dca 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -6,13 +6,14 @@ import ( "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -32,7 +33,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -54,7 +55,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -83,3 +84,9 @@ func TestClusterbombGenerator(t *testing.T) { } require.Equal(t, 3, count, "could not get correct clusterbomb counts") } + +func getOptions(allowLocalFileAccess bool) *types.Options { + opts := types.DefaultOptions() + opts.AllowLocalFileAccess = allowLocalFileAccess + return opts +} diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 92ec932019..390c8141e4 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,7 +2,7 @@ package generators import ( "bufio" - "path/filepath" + "io" "strings" "github.com/pkg/errors" @@ -11,7 +11,7 @@ import ( ) // loadPayloads loads the input payloads from a map to a data map -func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, allowLocalFileAccess bool) (map[string][]string, error) { +func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath string) (map[string][]string, error) { loadedPayloads := make(map[string][]string) for name, payload := range payloads { @@ -22,18 +22,11 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, if len(elements) >= 2 { loadedPayloads[name] = elements } else { - if !allowLocalFileAccess { - pt = filepath.Clean(pt) - templateAbsPath, err := filepath.Abs(templatePath) - if err != nil { - return nil, errors.Wrap(err, "could not get absolute path") - } - templatePathDir := filepath.Dir(templateAbsPath) - if !(templatePathDir != "/" && strings.HasPrefix(pt, templatePathDir)) && !strings.HasPrefix(pt, templateDirectory) { - return nil, errors.New("denied payload file path specified") - } + file, err := generator.options.LoadHelperFile(pt, templatePath, generator.catalog) + if err != nil { + return nil, errors.Wrap(err, "could not load payload file") } - payloads, err := generator.loadPayloadsFromFile(pt) + payloads, err := generator.loadPayloadsFromFile(file) if err != nil { return nil, errors.Wrap(err, "could not load payloads") } @@ -47,13 +40,8 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, } // loadPayloadsFromFile loads a file to a string slice -func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) { +func (generator *PayloadGenerator) loadPayloadsFromFile(file io.ReadCloser) ([]string, error) { var lines []string - - file, err := generator.catalog.OpenFile(filepath) - if err != nil { - return nil, err - } defer file.Close() scanner := bufio.NewScanner(file) diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index 28803b097b..49c00ef2f9 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -2,66 +2,119 @@ package generators import ( "os" + "os/exec" "path/filepath" "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" osutils "github.com/projectdiscovery/utils/os" "github.com/stretchr/testify/require" ) func TestLoadPayloads(t *testing.T) { - tempdir, err := os.MkdirTemp("", "templates-*") - require.NoError(t, err, "could not create temp dir") - defer os.RemoveAll(tempdir) + // since we are changing value of global variable i.e templates directory + // run this test as subprocess + if os.Getenv("LOAD_PAYLOAD_NO_ACCESS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess") + cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_NO_ACCESS=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + if err != nil { + t.Fatalf("process ran with err %v, want exit status 1", err) + } + } + templateDir := getTemplatesDir(t) + config.DefaultConfig.SetTemplatesDir(templateDir) - generator := &PayloadGenerator{catalog: disk.NewCatalog(tempdir)} - - fullpath := filepath.Join(tempdir, "payloads.txt") - err = os.WriteFile(fullpath, []byte("test\nanother"), 0777) - require.NoError(t, err, "could not write payload") + generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(false)} + fullpath := filepath.Join(templateDir, "payloads.txt") // Test sandbox t.Run("templates-directory", func(t *testing.T) { + // testcase when loading file from template directory and template file is in root + // expected to succeed values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/test", tempdir, false) + }, "/test") require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) t.Run("templates-path-relative", func(t *testing.T) { + // testcase when loading file from template directory and template file is current working directory + // expected to fail since this is LFI _, err := generator.loadPayloads(map[string]interface{}{ "new": "../../../../../../../../../etc/passwd", - }, ".", tempdir, false) + }, ".") require.Error(t, err, "could load payloads") }) t.Run("template-directory", func(t *testing.T) { + // testcase when loading file from template directory and template file is inside template directory + // expected to succeed values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, filepath.Join(tempdir, "test.yaml"), "/test", false) + }, filepath.Join(templateDir, "test.yaml")) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) - t.Run("no-sandbox-unix", func(t *testing.T) { - if osutils.IsWindows() { - return - } - _, err := generator.loadPayloads(map[string]interface{}{ - "new": "/etc/passwd", - }, "/random", "/test", true) - require.NoError(t, err, "could load payloads") - }) + t.Run("invalid", func(t *testing.T) { + // testcase when loading file from /etc/passwd and template file is at root i.e / + // expected to fail since this is LFI values, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", false) + }, "/random") require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") + // testcase when loading file from template directory and template file is at root i.e / + // expected to succeed values, err = generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/random", "/test", false) - require.Error(t, err, "could load payloads") - require.Equal(t, 0, len(values), "could get values") + }, "/random") + require.NoError(t, err, "could load payloads %v", values) + require.Equal(t, 1, len(values), "could get values") + require.Equal(t, []string{"test", "another"}, values["new"], "could get values") + }) +} + +func TestLoadPayloadsWithAccess(t *testing.T) { + // since we are changing value of global variable i.e templates directory + // run this test as subprocess + if os.Getenv("LOAD_PAYLOAD_WITH_ACCESS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess") + cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_WITH_ACCESS=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + if err != nil { + t.Fatalf("process ran with err %v, want exit status 1", err) + } + } + templateDir := getTemplatesDir(t) + config.DefaultConfig.SetTemplatesDir(templateDir) + + generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(true)} + + t.Run("no-sandbox-unix", func(t *testing.T) { + if osutils.IsWindows() { + return + } + _, err := generator.loadPayloads(map[string]interface{}{ + "new": "/etc/passwd", + }, "/random") + require.NoError(t, err, "could load payloads") }) } + +func getTemplatesDir(t *testing.T) string { + tempdir, err := os.MkdirTemp("", "templates-*") + require.NoError(t, err, "could not create temp dir") + fullpath := filepath.Join(tempdir, "payloads.txt") + err = os.WriteFile(fullpath, []byte("test\nanother"), 0777) + require.NoError(t, err, "could not write payload") + return tempdir +} diff --git a/v2/pkg/protocols/common/variables/variables.go b/v2/pkg/protocols/common/variables/variables.go index b806402aaa..7a7fda1f03 100644 --- a/v2/pkg/protocols/common/variables/variables.go +++ b/v2/pkg/protocols/common/variables/variables.go @@ -66,6 +66,11 @@ func (variables *Variable) UnmarshalJSON(data []byte) error { func (variables *Variable) Evaluate(values map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}, variables.Len()) variables.ForEach(func(key string, value interface{}) { + if sliceValue, ok := value.([]interface{}); ok { + // slices cannot be evaluated + result[key] = sliceValue + return + } valueString := types.ToString(value) combined := generators.MergeMaps(values, result) if value, ok := combined[key]; ok { @@ -76,12 +81,26 @@ func (variables *Variable) Evaluate(values map[string]interface{}) map[string]in return result } +// GetAll returns all variables as a map +func (variables *Variable) GetAll() map[string]interface{} { + result := make(map[string]interface{}, variables.Len()) + variables.ForEach(func(key string, value interface{}) { + result[key] = value + }) + return result +} + // EvaluateWithInteractsh returns evaluation results of variables with interactsh func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{}, interact *interactsh.Client) (map[string]interface{}, []string) { result := make(map[string]interface{}, variables.Len()) var interactURLs []string variables.ForEach(func(key string, value interface{}) { + if sliceValue, ok := value.([]interface{}); ok { + // slices cannot be evaluated + result[key] = sliceValue + return + } valueString := types.ToString(value) if strings.Contains(valueString, "interactsh-url") { valueString, interactURLs = interact.Replace(valueString, interactURLs) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 1cc8302e57..232427058a 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -172,7 +172,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 8d05d37172..c2e1fcdbed 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) // merge with metadata (eg. from workflow context) - vars = generators.MergeMaps(vars, metadata, optionVars, request.options.TemplateCtx.GetAll()) + vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(vars, variablesMap, request.options.Constants) @@ -66,18 +66,18 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, break } value = generators.MergeMaps(vars, value) - if err := request.execute(domain, metadata, previous, value, callback); err != nil { + if err := request.execute(input, domain, metadata, previous, value, callback); err != nil { return err } } } else { value := maps.Clone(vars) - return request.execute(domain, metadata, previous, value, callback) + return request.execute(input, domain, metadata, previous, value, callback) } return nil } -func (request *Request) execute(domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { +func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { if vardump.EnableVarDump { gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) @@ -151,7 +151,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData) // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) for k, v := range previous { outputEvent[k] = v } @@ -159,7 +159,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent[k] = v } // add variables from template context before matching/extraction - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) dumpResponse(event, request, request.options, response.String(), question) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index dc19deccf6..324b5a1946 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -64,7 +64,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // every new file in the compressed multi-file archive counts 1 request.options.Progress.AddToTotal(1) archiveFileName := filepath.Join(filePath, file.Name()) - event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous) + event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -117,7 +117,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, _ = tmpFileOut.Sync() // rewind the file _, _ = tmpFileOut.Seek(0, 0) - event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous) + event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -137,7 +137,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, default: // normal file - increments the counter by 1 request.options.Progress.AddToTotal(1) - event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous) + event, fileMatches, err := request.processFile(filePath, input, previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -166,7 +166,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, return nil } -func (request *Request) processFile(filePath, input string, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { +func (request *Request) processFile(filePath string, input *contextargs.Context, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { file, err := os.Open(filePath) if err != nil { return nil, nil, errors.Errorf("Could not open file path %s: %s\n", filePath, err) @@ -185,7 +185,7 @@ func (request *Request) processFile(filePath, input string, previousInternalEven return request.processReader(file, filePath, input, stat.Size(), previousInternalEvent) } -func (request *Request) processReader(reader io.Reader, filePath, input string, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { +func (request *Request) processReader(reader io.Reader, filePath string, input *contextargs.Context, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { fileReader := io.LimitReader(reader, request.maxSize) fileMatches, opResult := request.findMatchesWithReader(fileReader, input, filePath, totalBytes, previousInternalEvent) if opResult == nil && len(fileMatches) == 0 { @@ -193,10 +193,10 @@ func (request *Request) processReader(reader io.Reader, filePath, input string, } // build event structure to interface with internal logic - return request.buildEvent(input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil + return request.buildEvent(input.MetaInput.Input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil } -func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) { +func (request *Request) findMatchesWithReader(reader io.Reader, input *contextargs.Context, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) { var bytesCount, linesCount, wordsCount int isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse totalBytesString := units.BytesSize(float64(totalBytes)) @@ -243,12 +243,12 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath processedBytes := units.BytesSize(float64(currentBytes)) gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytesString) - dslMap := request.responseToDSLMap(lineContent, input, filePath) + dslMap := request.responseToDSLMap(lineContent, input.MetaInput.Input, filePath) for k, v := range previous { dslMap[k] = v } // add template context variables to DSL map - dslMap = generators.MergeMaps(dslMap, request.options.TemplateCtx.GetAll()) + dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll()) discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug) newOpResult := discardEvent.OperatorsResult if newOpResult != nil { diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3524b47d86..7930598d21 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -349,7 +349,7 @@ func TestActionGetResource(t *testing.T) {