diff --git a/cmd/integration-test/code.go b/cmd/integration-test/code.go index c57af85a54..ebe234aa85 100644 --- a/cmd/integration-test/code.go +++ b/cmd/integration-test/code.go @@ -3,6 +3,7 @@ package main import ( "errors" "log" + "os" "path/filepath" osutils "github.com/projectdiscovery/utils/os" @@ -12,14 +13,16 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/testutils" ) +var isCodeDisabled = func() bool { return osutils.IsWindows() && os.Getenv("CI") == "true" } + var codeTestCases = []TestCaseInfo{ - {Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}}, - {Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}}, - {Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}}, - {Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}}, - {Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}}, - {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}}, - {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() }}, + {Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, + {Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}, DisableOn: isCodeDisabled}, + {Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}, DisableOn: isCodeDisabled}, + {Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}, DisableOn: isCodeDisabled}, + {Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled}, + {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, + {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }}, } const ( @@ -30,6 +33,10 @@ const ( var testcertpath = "" func init() { + if isCodeDisabled() { + // skip executing code protocol in CI on windows + return + } // allow local file access to load content of file references in template // in order to sign them for testing purposes templates.TemplateSignerLFA() diff --git a/cmd/integration-test/flow.go b/cmd/integration-test/flow.go index e57ede3708..097a1c5140 100644 --- a/cmd/integration-test/flow.go +++ b/cmd/integration-test/flow.go @@ -15,6 +15,7 @@ var flowTestcases = []TestCaseInfo{ {Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, {Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, {Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, + {Path: "flow/flow-hide-matcher.yaml", TestCase: &flowHideMatcher{}}, } type conditionalFlow struct{} @@ -24,7 +25,7 @@ func (t *conditionalFlow) Execute(filePath string) error { if err != nil { return err } - return expectResultsCount(results, 2) + return expectResultsCount(results, 1) } type conditionalFlowNegative struct{} @@ -66,7 +67,7 @@ func (t *iterateValuesFlow) Execute(filePath string) error { if err != nil { return err } - return expectResultsCount(results, 2) + return expectResultsCount(results, 1) } type dnsNsProbe struct{} @@ -76,9 +77,20 @@ func (t *dnsNsProbe) Execute(filePath string) error { if err != nil { return err } - return expectResultsCount(results, 3) + return expectResultsCount(results, 1) } func getBase64(input string) string { return base64.StdEncoding.EncodeToString([]byte(input)) } + +type flowHideMatcher struct{} + +func (t *flowHideMatcher) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + // this matcher should not return any results + return expectResultsCount(results, 0) +} diff --git a/cmd/integration-test/javascript.go b/cmd/integration-test/javascript.go index bdb511b07e..e45f122c30 100644 --- a/cmd/integration-test/javascript.go +++ b/cmd/integration-test/javascript.go @@ -14,6 +14,7 @@ var jsTestcases = []TestCaseInfo{ {Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, {Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, {Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}}, + {Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}}, } var ( @@ -23,6 +24,16 @@ var ( defaultRetry = 3 ) +type javascriptNetHttps struct{} + +func (j *javascriptNetHttps) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + type javascriptRedisPassBrute struct{} func (j *javascriptRedisPassBrute) Execute(filePath string) error { diff --git a/cmd/integration-test/network.go b/cmd/integration-test/network.go index 5fd2d0fb16..8081c973e9 100644 --- a/cmd/integration-test/network.go +++ b/cmd/integration-test/network.go @@ -1,11 +1,15 @@ package main import ( + "fmt" "net" + "os" "strings" + "time" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" osutils "github.com/projectdiscovery/utils/os" + "github.com/projectdiscovery/utils/reader" ) var networkTestcases = []TestCaseInfo{ @@ -16,6 +20,8 @@ var networkTestcases = []TestCaseInfo{ {Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}}, {Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}}, {Path: "protocols/network/network-port.yaml", TestCase: &networkPort{}}, + {Path: "protocols/network/net-https.yaml", TestCase: &networkhttps{}}, + {Path: "protocols/network/net-https-timeout.yaml", TestCase: &networkhttps{}}, } const defaultStaticPort = 5431 @@ -29,22 +35,26 @@ func (h *networkBasic) Execute(filePath string) error { ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { defer conn.Close() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { + data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) + if err != nil { routerErr = err return } if string(data) == "PING" { _, _ = conn.Write([]byte("PONG")) + } else { + routerErr = fmt.Errorf("invalid data received: %s", string(data)) } }) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { + fmt.Fprintf(os.Stderr, "Could not run nuclei: %s\n", err) return err } if routerErr != nil { + fmt.Fprintf(os.Stderr, "routerErr: %s\n", routerErr) return routerErr } @@ -60,8 +70,8 @@ func (h *networkMultiStep) Execute(filePath string) error { ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { defer conn.Close() - data := make([]byte, 5) - if _, err := conn.Read(data); err != nil { + data, err := reader.ConnReadNWithTimeout(conn, 5, time.Duration(5)*time.Second) + if err != nil { routerErr = err return } @@ -69,8 +79,8 @@ func (h *networkMultiStep) Execute(filePath string) error { _, _ = conn.Write([]byte("PING")) } - data = make([]byte, 6) - if _, err := conn.Read(data); err != nil { + data, err = reader.ConnReadNWithTimeout(conn, 6, time.Duration(5)*time.Second) + if err != nil { routerErr = err return } @@ -126,8 +136,8 @@ func (h *networkVariables) Execute(filePath string) error { ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { defer conn.Close() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { + data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) + if err != nil { routerErr = err return } @@ -154,8 +164,8 @@ func (n *networkPort) Execute(filePath string) error { ts := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) { defer conn.Close() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { + data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) + if err != nil { return } if string(data) == "PING" { @@ -187,8 +197,8 @@ func (n *networkPort) Execute(filePath string) error { ts2 := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) { defer conn.Close() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { + data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) + if err != nil { return } if string(data) == "PING" { @@ -206,3 +216,14 @@ func (n *networkPort) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type networkhttps struct{} + +// Execute executes a test case and returns an error if occurred +func (h *networkhttps) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} diff --git a/go.mod b/go.mod index cae1fac027..689709e310 100644 --- a/go.mod +++ b/go.mod @@ -91,7 +91,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766 github.com/projectdiscovery/uncover v1.0.7 - github.com/projectdiscovery/utils v0.0.58 + github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6 github.com/projectdiscovery/wappalyzergo v0.0.109 github.com/redis/go-redis/v9 v9.1.0 github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02 @@ -166,7 +166,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect - github.com/minio/selfupdate v0.6.0 // indirect + github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.5.0 // indirect diff --git a/go.sum b/go.sum index 974f355356..df0b78baa5 100644 --- a/go.sum +++ b/go.sum @@ -679,8 +679,8 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= -github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= -github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= +github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= +github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -722,8 +722,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -837,8 +837,8 @@ github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766 h1:wa2wak7 github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766/go.mod h1:bFATagikCvdPOsmaN1h5VQSbZjTW8bCQ6bjoQEePUq8= github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= -github.com/projectdiscovery/utils v0.0.58 h1:kk2AkSO84QZc9rDRI8jWA2Iia4uzb4sUcfh4h0xA20I= -github.com/projectdiscovery/utils v0.0.58/go.mod h1:rsR5Kzjrb+/Yp7JSnEblLk4LfU4zH5Z7wQn8RzaGSdY= +github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6 h1:60DKG3aueYiy93ZPt78yyZW0N+b7pWbK8Ub1UH6o08I= +github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6/go.mod h1:vt4oY4rvRWTdkBMhLlAGPbapa/R8pa+xZBYuNZIKJgQ= github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k= github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/integration_tests/flow/flow-hide-matcher.yaml b/integration_tests/flow/flow-hide-matcher.yaml new file mode 100644 index 0000000000..f8ffc27184 --- /dev/null +++ b/integration_tests/flow/flow-hide-matcher.yaml @@ -0,0 +1,28 @@ +id: flow-hide-matcher + +info: + name: Test HTTP Template + author: pdteam + severity: info + description: In flow matcher output of previous step is hidden and only last event matcher output is shown + +flow: http(1) && http(2) + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - ok + + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "Failed event" \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index e3487bd94a..c4b9e6c5b7 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4a0a00473045022100d407a3b848664b4c271abb4462a89a53fa2da6c21fd66011974ac395e2dc041c0220129a752a792337f6efe2e96562989016fe2709820b9583fd933f02be3b9d074f:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index f18b7bb2ba..74203e8df3 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a004830460221009db4541aa2af10aae5f39fe6e8789e2717c96ebbdadfdf33114ec0e82ec4da73022100fa98ee6611b606befc139946a169cca717f16ebf71beac97fdde1fe0c7fba774:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 96e16e98e4..eac53c3bea 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4a0a0047304502205ebee72972ea0005ecdbcf7cd676ab861f3a44477a4b85dc1e745b7a628d2d7a022100ec4604673a1d43311ab343005464be5d4ee26b5a1f39206aa841056f3e2057dd:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 7a6a1781b4..f8a30b4f1f 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -21,4 +21,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a004830460221009a87b77e770e688bb1ce05e75ac075cdb3f318aad18a6dbc3fc2ec729a8ba5990221009020d69ba3baf47f9d835d4b6bd644a9e4f2d699369acc2a15983f5c270d2e79:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/javascript/net-https.yaml b/integration_tests/protocols/javascript/net-https.yaml new file mode 100644 index 0000000000..9761bc9567 --- /dev/null +++ b/integration_tests/protocols/javascript/net-https.yaml @@ -0,0 +1,25 @@ +id: net-https + +info: + name: net-https + author: pdteam + severity: info + description: send and receive https data using net module + + +javascript: + - code: | + let m = require('nuclei/net'); + let name=Host+':'+Port; + let conn = m.OpenTLS('tcp', name); + conn.Send('GET / HTTP/1.1\r\nHost:'+name+'\r\nConnection: close\r\n\r\n'); + resp = conn.RecvString(); + + args: + Host: "{{Host}}" + Port: "443" + + matchers: + - type: word + words: + - "HTTP/1.1 200 OK" \ No newline at end of file diff --git a/integration_tests/protocols/network/net-https-timeout.yaml b/integration_tests/protocols/network/net-https-timeout.yaml new file mode 100644 index 0000000000..a0d106fc0b --- /dev/null +++ b/integration_tests/protocols/network/net-https-timeout.yaml @@ -0,0 +1,25 @@ +id: net-https-timeout + +info: + name: Example Network template which times out + author: pdteam + severity: high + description: Example Network template to send HTTPS request which times out + + +tcp: + - host: + - "tls://{{Hostname}}" + port: 443 + inputs: + # noticable difference between this and net-https.yaml is that here we don't send the Connection: close header + # and hence connection will remain open until server closes it. This can be a DOS vector in nuclei + # as it waits for server to close the connection. now we have set a default timeout of 5 seconds and if server responds but doesn't close the connection + # then nuclei will close connection but doesn't fail the request since we already have response data from server + # this feature is only required for `read-all: true` to work properly + - data: "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n" + read-all: true + extractors: + - type: dsl + dsl: + - "len(data)" \ No newline at end of file diff --git a/integration_tests/protocols/network/net-https.yaml b/integration_tests/protocols/network/net-https.yaml new file mode 100644 index 0000000000..94914560f2 --- /dev/null +++ b/integration_tests/protocols/network/net-https.yaml @@ -0,0 +1,20 @@ +id: net-https + +info: + name: Example Network template to send HTTPS request + author: pdteam + severity: high + description: Example Network template to send HTTPS request + + +tcp: + - host: + - "tls://{{Hostname}}" + port: 443 + inputs: + - data: "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\nConnection: close\r\n\r\n" + read-all: true + extractors: + - type: dsl + dsl: + - "len(data)" \ No newline at end of file diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 4d320b4c90..2b913d939f 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -692,7 +692,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { if k != templates.Unsigned { gologger.Info().Msgf("Executing %d signed templates from %s", v.Load(), k) } else if !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { - gologger.DefaultLogger.Print().Msgf("[%v] Executing %d unsigned templates. Use with caution.", aurora.BrightYellow("WRN"), v.Load()) + gologger.Print().Msgf("[%v] Executing %d unsigned templates. Use with caution.", aurora.BrightYellow("WRN"), v.Load()) } } } diff --git a/lib/example_test.go b/lib/example_test.go index 22f614f8f7..1bbc20e584 100644 --- a/lib/example_test.go +++ b/lib/example_test.go @@ -72,6 +72,7 @@ func ExampleThreadSafeNucleiEngine() { // Output: // [nameserver-fingerprint] scanme.sh + // [caa-fingerprint] honey.scanme.sh } func TestMain(m *testing.M) { diff --git a/pkg/js/generated/js/libnet/net.js b/pkg/js/generated/js/libnet/net.js index 50b84e3532..40e830b5d5 100644 --- a/pkg/js/generated/js/libnet/net.js +++ b/pkg/js/generated/js/libnet/net.js @@ -20,8 +20,8 @@ class NetConn { /** * @method - * @description Recv receives data from the connection with a timeout. If N is 0, it will read up to 4096 bytes. - * @param {number} N - The number of bytes to receive. + * @description Recv receives data from the connection with a timeout. If N is 0, it will read all available data. + * @param {number} [N=0] - The number of bytes to receive. * @returns {Uint8Array} - The received data in an array. * @throws {error} - The error encountered during data receiving. * @example @@ -35,8 +35,8 @@ class NetConn { /** * @method - * @description RecvHex receives data from the connection with a timeout in hex format. If N is 0, it will read up to 4096 bytes. - * @param {number} N - The number of bytes to receive. + * @description RecvHex receives data from the connection with a timeout in hex format. If N is 0, it will read all available data. + * @param {number} [N=0] - The number of bytes to receive. * @returns {string} - The received data in hex format. * @throws {error} - The error encountered during data receiving. * @example @@ -50,8 +50,8 @@ class NetConn { /** * @method - * @description RecvString receives data from the connection with a timeout. Output is returned as a string. If N is 0, it will read up to 4096 bytes. - * @param {number} N - The number of bytes to receive. + * @description RecvString receives data from the connection with a timeout. Output is returned as a string. If N is 0, it will read all available data. + * @param {number} [N=0] - The number of bytes to receive. * @returns {string} - The received data as a string. * @throws {error} - The error encountered during data receiving. * @example diff --git a/pkg/js/libs/net/net.go b/pkg/js/libs/net/net.go index dd3a8dc4ef..7d5b721719 100644 --- a/pkg/js/libs/net/net.go +++ b/pkg/js/libs/net/net.go @@ -4,14 +4,18 @@ import ( "context" "crypto/tls" "encoding/hex" - "errors" "fmt" "net" - "syscall" "time" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" + "github.com/projectdiscovery/utils/reader" +) + +var ( + defaultTimeout = time.Duration(5) * time.Second ) // Open opens a new connection to the address with a timeout. @@ -21,13 +25,13 @@ func Open(protocol, address string) (*NetConn, error) { if err != nil { return nil, err } - return &NetConn{conn: conn}, nil + return &NetConn{conn: conn, timeout: defaultTimeout}, nil } // Open opens a new connection to the address with a timeout. // supported protocols: tcp, udp func OpenTLS(protocol, address string) (*NetConn, error) { - config := &tls.Config{InsecureSkipVerify: true} + config := &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10} host, _, _ := net.SplitHostPort(address) if host != "" { c := config.Clone() @@ -38,7 +42,7 @@ func OpenTLS(protocol, address string) (*NetConn, error) { if err != nil { return nil, err } - return &NetConn{conn: conn}, nil + return &NetConn{conn: conn, timeout: defaultTimeout}, nil } // NetConn is a connection to a remote host. @@ -67,9 +71,15 @@ func (c *NetConn) setDeadLine() { _ = c.conn.SetDeadline(time.Now().Add(c.timeout)) } +// unsetDeadLine unsets read/write deadline for the connection. +func (c *NetConn) unsetDeadLine() { + _ = c.conn.SetDeadline(time.Time{}) +} + // SendArray sends array data to connection func (c *NetConn) SendArray(data []interface{}) error { c.setDeadLine() + defer c.unsetDeadLine() input := types.ToByteSlice(data) length, err := c.conn.Write(input) if err != nil { @@ -84,6 +94,7 @@ func (c *NetConn) SendArray(data []interface{}) error { // SendHex sends hex data to connection func (c *NetConn) SendHex(data string) error { c.setDeadLine() + defer c.unsetDeadLine() bin, err := hex.DecodeString(data) if err != nil { return err @@ -101,6 +112,7 @@ func (c *NetConn) SendHex(data string) error { // Send sends data to the connection with a timeout. func (c *NetConn) Send(data string) error { c.setDeadLine() + defer c.unsetDeadLine() bin := []byte(data) length, err := c.conn.Write(bin) if err != nil { @@ -113,30 +125,24 @@ func (c *NetConn) Send(data string) error { } // Recv receives data from the connection with a timeout. -// If N is 0, it will read up to 4096 bytes. +// If N is 0, it will read all data sent by the server with 8MB limit. func (c *NetConn) Recv(N int) ([]byte, error) { c.setDeadLine() - var response []byte - if N > 0 { - response = make([]byte, N) - } else { - response = make([]byte, 4096) + defer c.unsetDeadLine() + if N == 0 { + // in utils we use -1 to indicate read all rather than 0 + N = -1 } - length, err := c.conn.Read(response) + bin, err := reader.ConnReadNWithTimeout(c.conn, int64(N), c.timeout) if err != nil { - var netErr net.Error - if (errors.As(err, &netErr) && netErr.Timeout()) || - errors.Is(err, syscall.ECONNREFUSED) { // timeout error or connection refused - return response, nil - } - return response[:length], err + return []byte{}, errorutil.NewWithErr(err).Msgf("failed to read %d bytes", N) } - return response[:length], nil + return bin, nil } // RecvString receives data from the connection with a timeout // output is returned as a string. -// If N is 0, it will read up to 4096 bytes. +// If N is 0, it will read all data sent by the server with 8MB limit. func (c *NetConn) RecvString(N int) (string, error) { bin, err := c.Recv(N) if err != nil { @@ -147,7 +153,7 @@ func (c *NetConn) RecvString(N int) (string, error) { // RecvHex receives data from the connection with a timeout // in hex format. -// If N is 0, it will read up to 4096 bytes. +// If N is 0,it will read all data sent by the server with 8MB limit. func (c *NetConn) RecvHex(N int) (string, error) { bin, err := c.Recv(N) if err != nil { diff --git a/pkg/js/libs/smb/smbghost.go b/pkg/js/libs/smb/smbghost.go index 275eaee117..7329606c90 100644 --- a/pkg/js/libs/smb/smbghost.go +++ b/pkg/js/libs/smb/smbghost.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/structs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/utils/reader" ) const ( @@ -31,10 +32,8 @@ func (c *SMBClient) DetectSMBGhost(host string, port int) (bool, error) { if err != nil { return false, err } - - buff := make([]byte, 4) - nb, _ := conn.Read(buff) - args, err := structs.Unpack(">I", buff[:nb]) + buff, _ := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) + args, err := structs.Unpack(">I", buff) if err != nil { return false, err } @@ -43,13 +42,14 @@ func (c *SMBClient) DetectSMBGhost(host string, port int) (bool, error) { } length := args[0].(int) - data := make([]byte, length) _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) - n, err := conn.Read(data) + data, err := reader.ConnReadNWithTimeout(conn, int64(length), time.Duration(5)*time.Second) if err != nil { return false, err } - data = data[:n] + if len(data) < 72 { + return false, errors.New("invalid response expected at least 72 bytes") + } if !bytes.Equal(data[68:70], []byte("\x11\x03")) || !bytes.Equal(data[70:72], []byte("\x02\x00")) { return false, nil diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 4fb15cd08d..08a7865430 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -435,7 +435,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte request.options.Progress.IncrementRequests() requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err) - gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.TemplateID, hostPort) + gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { msg := fmt.Sprintf("[%s] Dumped Javascript request for %s:\nVariables:\n %v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(argsCopy.Args)) diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 6fd9cbedff..4898478495 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -4,10 +4,8 @@ import ( "context" "encoding/hex" "fmt" - "io" "net" "net/url" - "os" "strings" "time" @@ -30,6 +28,13 @@ import ( templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" errorutil "github.com/projectdiscovery/utils/errors" mapsutil "github.com/projectdiscovery/utils/maps" + "github.com/projectdiscovery/utils/reader" +) + +var ( + // TODO: make this configurable + // DefaultReadTimeout is the default read timeout for network requests + DefaultReadTimeout = time.Duration(5) * time.Second ) var _ protocols.Request = &Request{} @@ -196,15 +201,14 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac } if input.Read > 0 { - buffer := make([]byte, input.Read) - n, err := conn.Read(buffer) + buffer, err := reader.ConnReadNWithTimeout(conn, int64(input.Read), DefaultReadTimeout) if err != nil { return errorutil.NewWithErr(err).Msgf("could not read response from connection") } - responseBuilder.Write(buffer[:n]) + responseBuilder.Write(buffer) - bufferStr := string(buffer[:n]) + bufferStr := string(buffer) if input.Name != "" { inputEvents[input.Name] = bufferStr interimValues[input.Name] = bufferStr @@ -243,51 +247,19 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac if request.ReadSize != 0 { bufferSize = request.ReadSize } - - var ( - final []byte - n int - ) - if request.ReadAll { - readInterval := time.NewTimer(time.Second * 1) - // stop the timer and drain the channel - closeTimer := func(t *time.Timer) { - if !t.Stop() { - <-t.C - } - } - readSocket: - for { - select { - case <-readInterval.C: - closeTimer(readInterval) - break readSocket - default: - buf := make([]byte, bufferSize) - nBuf, err := conn.Read(buf) - if err != nil && !os.IsTimeout(err) && err != io.EOF { - request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) - closeTimer(readInterval) - return errors.Wrap(err, "could not read from server") - } - responseBuilder.Write(buf[:nBuf]) - final = append(final, buf...) - n += nBuf - } - } - } else { - final = make([]byte, bufferSize) - n, err = conn.Read(final) - if err != nil && !os.IsTimeout(err) && err != io.EOF { - request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) - return errors.Wrap(err, "could not read from server") - } - responseBuilder.Write(final[:n]) + bufferSize = -1 + } + + final, err := reader.ConnReadNWithTimeout(conn, int64(bufferSize), DefaultReadTimeout) + if err != nil { + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) + return errors.Wrap(err, "could not read from server") } + responseBuilder.Write(final) response := responseBuilder.String() - outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input.MetaInput.Input, actualAddress) + outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final), response, input.MetaInput.Input, actualAddress) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) diff --git a/pkg/templates/types/types.go b/pkg/templates/types/types.go index 6d26530de0..b8987ec322 100644 --- a/pkg/templates/types/types.go +++ b/pkg/templates/types/types.go @@ -58,7 +58,7 @@ var protocolMappings = map[ProtocolType]string{ WebsocketProtocol: "websocket", WHOISProtocol: "whois", CodeProtocol: "code", - JavascriptProtocol: "js", + JavascriptProtocol: "javascript", } func GetSupportedProtocolTypes() ProtocolTypes { diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 1bb5ca8129..04394744c0 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -5,7 +5,6 @@ import ( "io" "strconv" "strings" - "sync" "sync/atomic" "github.com/dop251/goja" @@ -29,22 +28,28 @@ var ( ErrInvalidRequestID = errorutil.NewWithFmt("[%s] invalid request id '%s' provided") ) +// ProtoOptions are options that can be passed to flow protocol callback +// ex: dns(protoOptions) <- protoOptions are optional and can be anything +type ProtoOptions struct { + protoName string + reqIDS []string +} + // FlowExecutor is a flow executor for executing a flow type FlowExecutor struct { input *contextargs.Context options *protocols.ExecutorOptions // javascript runtime reference and compiled program - jsVM *goja.Runtime - program *goja.Program // compiled js program + jsVM *goja.Runtime + program *goja.Program // compiled js program + lastEvent *output.InternalWrappedEvent // contains last event that was emitted // protocol requests and their callback functions allProtocols map[string][]protocols.Request protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js - callback func(event *output.InternalWrappedEvent) // result event callback // logic related variables - wg sync.WaitGroup results *atomic.Bool allErrs mapsutil.SyncLockMap[string, error] } @@ -72,6 +77,8 @@ func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, o allprotos[templateTypes.WHOISProtocol.String()] = append(allprotos[templateTypes.WHOISProtocol.String()], req) case templateTypes.CodeProtocol: allprotos[templateTypes.CodeProtocol.String()] = append(allprotos[templateTypes.CodeProtocol.String()], req) + case templateTypes.JavascriptProtocol: + allprotos[templateTypes.JavascriptProtocol.String()] = append(allprotos[templateTypes.JavascriptProtocol.String()], req) default: gologger.Error().Msgf("invalid request type %s", req.Type().String()) } @@ -143,22 +150,10 @@ func (f *FlowExecutor) Compile() error { } for _, v := range call.Arguments { switch value := v.Export().(type) { - case map[string]interface{}: - opts.LoadOptions(value) default: opts.reqIDS = append(opts.reqIDS, types.ToString(value)) } } - // parallel execution of protocols - if opts.Async { - f.wg.Add(1) - go func() { - defer f.wg.Done() - f.requestExecutor(reqMap, opts) - }() - return f.jsVM.ToValue(true) - } - return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) } } @@ -174,7 +169,6 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p } }() - f.callback = callback f.input = input // -----Load all types of variables----- // add all input args to template context @@ -183,7 +177,7 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value) }) } - if f.callback == nil { + if callback == nil { return fmt.Errorf("output callback cannot be nil") } // pass flow and execute the js vm and handle errors @@ -191,11 +185,12 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p if err != nil { return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) } - f.wg.Wait() runtimeErr := f.GetRuntimeErrors() if runtimeErr != nil { return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") } + // this is where final result is generated/created + callback(f.lastEvent) if value.Export() != nil { f.results.Store(value.ToBoolean()) } else { diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go index a073422d30..b435e92728 100644 --- a/pkg/tmplexec/flow/flow_internal.go +++ b/pkg/tmplexec/flow/flow_internal.go @@ -34,34 +34,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req // execution logic for http()/dns() etc for index := range f.allProtocols[opts.protoName] { req := f.allProtocols[opts.protoName][index] - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) { - if result != nil { - f.results.CompareAndSwap(false, true) - if !opts.Hide { - f.callback(result) - } - // export dynamic values from operators (i.e internal:true) - // add add it to template context - // this is a conflicting behaviour with iterate-all - if result.HasOperatorResult() { - matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) - if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { - // if matcher status is false . check if template/request contains any matcher at all - // if it does then we need to set matcher status to true - matcherStatus.CompareAndSwap(false, true) - } - if len(result.OperatorsResult.DynamicValues) > 0 { - for k, v := range result.OperatorsResult.DynamicValues { - f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) - } - } - } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { - // if matcher status is false . check if template/request contains any matcher at all - // if it does then we need to set matcher status to true - matcherStatus.CompareAndSwap(false, true) - } - } - }) + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, f.getProtoRequestCallback(req, matcherStatus, opts)) if err != nil { // save all errors in a map with id as key // its less likely that there will be race condition but just in case @@ -90,25 +63,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req } return matcherStatus.Load() } - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) { - if result != nil { - f.results.CompareAndSwap(false, true) - if !opts.Hide { - f.callback(result) - } - // export dynamic values from operators (i.e internal:true) - // add add it to template context - if result.HasOperatorResult() { - matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) - if len(result.OperatorsResult.DynamicValues) > 0 { - for k, v := range result.OperatorsResult.DynamicValues { - f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) - } - _ = f.jsVM.Set("template", f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) - } - } - } - }) + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, f.getProtoRequestCallback(req, matcherStatus, opts)) if err != nil { index := id err = f.allErrs.Set(opts.protoName+":"+index, err) @@ -120,6 +75,39 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req return matcherStatus.Load() } +// getProtoRequestCallback returns a callback that is executed +// after execution of each protocol request +func (f *FlowExecutor) getProtoRequestCallback(req protocols.Request, matcherStatus *atomic.Bool, opts *ProtoOptions) func(result *output.InternalWrappedEvent) { + return func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + f.lastEvent = result + // export dynamic values from operators (i.e internal:true) + // add add it to template context + // this is a conflicting behaviour with iterate-all + if result.HasOperatorResult() { + // this is to handle case where there is any operator result (matcher or extractor) + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) + } + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) + } + } + } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { + // this is to handle case where there are no operator result and there was no matcher in operators + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) + } + } + } +} + // registerBuiltInFunctions registers all built in functions for the flow func (f *FlowExecutor) registerBuiltInFunctions() error { // currently we register following builtin functions diff --git a/pkg/tmplexec/flow/options.go b/pkg/tmplexec/flow/options.go deleted file mode 100644 index dab7cdc9cc..0000000000 --- a/pkg/tmplexec/flow/options.go +++ /dev/null @@ -1,48 +0,0 @@ -package flow - -import ( - "strings" - - "github.com/projectdiscovery/nuclei/v3/pkg/types" -) - -// ProtoOptions are options that can be passed to flow protocol callback -// ex: dns(protoOptions) <- protoOptions are optional and can be anything -type ProtoOptions struct { - Hide bool - Async bool - protoName string - reqIDS []string -} - -// Examples -// dns() <- callback without any options -// dns(1) or dns(1,3) <- callback with index of protocol in template request at 1 or 1 and 3 -// dns("probe-http") or dns("extract-vpc","probe-http") <- callback with id's instead of index of request in template -// dns({hide:true}) or dns({hide:true,async:true}) <- callback with protocol options -// hide - hides result/event from output & sdk -// async - executes protocols in parallel (implicit wait no need to specify wait) -// Note: all of these options are optional and can be combined together in any order - -// LoadOptions loads the protocol options from a map -func (P *ProtoOptions) LoadOptions(m map[string]interface{}) { - P.Hide = GetBool(m["hide"]) - P.Async = GetBool(m["async"]) -} - -// GetBool returns bool value from interface -func GetBool(value interface{}) bool { - if value == nil { - return false - } - switch v := value.(type) { - case bool: - return v - default: - tmpValue := types.ToString(value) - if strings.EqualFold(tmpValue, "true") { - return true - } - } - return false -}