bug fixes in js , network protocol and flow

cmd/integration-test/code.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"log"
+	"os"
 	"path/filepath"
 
 	osutils "github.com/projectdiscovery/utils/os"
@@ -12,14 +13,16 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
+var isCodeDisabled = func() bool { return osutils.IsWindows() && os.Getenv("CI") == "true" }
+
 var codeTestCases = []TestCaseInfo{
-	{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}},
-	{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}},
-	{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}},
-	{Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}},
-	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}},
-	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}},
-	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() }},
+	{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
 }
 
 const (
@@ -30,6 +33,10 @@ const (
 var testcertpath = ""
 
 func init() {
+	if isCodeDisabled() {
+		// skip executing code protocol in CI on windows
+		return
+	}
 	// allow local file access to load content of file references in template
 	// in order to sign them for testing purposes
 	templates.TemplateSignerLFA()

cmd/integration-test/flow.go

@@ -15,6 +15,7 @@ var flowTestcases = []TestCaseInfo{
 	{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}},
 	{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}},
 	{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}},
+	{Path: "flow/flow-hide-matcher.yaml", TestCase: &flowHideMatcher{}},
 }
 
 type conditionalFlow struct{}
@@ -24,7 +25,7 @@ func (t *conditionalFlow) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 2)
+	return expectResultsCount(results, 1)
 }
 
 type conditionalFlowNegative struct{}
@@ -66,7 +67,7 @@ func (t *iterateValuesFlow) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 2)
+	return expectResultsCount(results, 1)
 }
 
 type dnsNsProbe struct{}
@@ -76,9 +77,20 @@ func (t *dnsNsProbe) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 3)
+	return expectResultsCount(results, 1)
 }
 
 func getBase64(input string) string {
 	return base64.StdEncoding.EncodeToString([]byte(input))
 }
+
+type flowHideMatcher struct{}
+
+func (t *flowHideMatcher) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	// this matcher should not return any results
+	return expectResultsCount(results, 0)
+}

cmd/integration-test/javascript.go

@@ -14,6 +14,7 @@ var jsTestcases = []TestCaseInfo{
 	{Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
+	{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
 }
 
 var (
@@ -23,6 +24,16 @@ var (
 	defaultRetry  = 3
 )
 
+type javascriptNetHttps struct{}
+
+func (j *javascriptNetHttps) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
 type javascriptRedisPassBrute struct{}
 
 func (j *javascriptRedisPassBrute) Execute(filePath string) error {

cmd/integration-test/network.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"fmt"
 	"net"
+	"os"
 	"strings"
+	"time"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 	osutils "github.com/projectdiscovery/utils/os"
+	"github.com/projectdiscovery/utils/reader"
 )
 
 var networkTestcases = []TestCaseInfo{
@@ -16,6 +20,8 @@ var networkTestcases = []TestCaseInfo{
 	{Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}},
 	{Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}},
 	{Path: "protocols/network/network-port.yaml", TestCase: &networkPort{}},
+	{Path: "protocols/network/net-https.yaml", TestCase: &networkhttps{}},
+	{Path: "protocols/network/net-https-timeout.yaml", TestCase: &networkhttps{}},
 }
 
 const defaultStaticPort = 5431
@@ -29,22 +35,26 @@ func (h *networkBasic) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
 		if string(data) == "PING" {
 			_, _ = conn.Write([]byte("PONG"))
+		} else {
+			routerErr = fmt.Errorf("invalid data received: %s", string(data))
 		}
 	})
 	defer ts.Close()
 
 	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not run nuclei: %s\n", err)
 		return err
 	}
 	if routerErr != nil {
+		fmt.Fprintf(os.Stderr, "routerErr: %s\n", routerErr)
 		return routerErr
 	}
 
@@ -60,17 +70,17 @@ func (h *networkMultiStep) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 5)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 5, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
 		if string(data) == "FIRST" {
 			_, _ = conn.Write([]byte("PING"))
 		}
 
-		data = make([]byte, 6)
-		if _, err := conn.Read(data); err != nil {
+		data, err = reader.ConnReadNWithTimeout(conn, 6, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -126,8 +136,8 @@ func (h *networkVariables) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -154,8 +164,8 @@ func (n *networkPort) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			return
 		}
 		if string(data) == "PING" {
@@ -187,8 +197,8 @@ func (n *networkPort) Execute(filePath string) error {
 	ts2 := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			return
 		}
 		if string(data) == "PING" {
@@ -206,3 +216,14 @@ func (n *networkPort) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type networkhttps struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *networkhttps) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}

go.mod

@@ -91,7 +91,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/utils v0.0.58
+	github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6
 	github.com/projectdiscovery/wappalyzergo v0.0.109
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02
@@ -166,7 +166,7 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
-	github.com/minio/selfupdate v0.6.0 // indirect
+	github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect

go.sum

@@ -679,8 +679,8 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
 github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
 github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
-github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
-github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
+github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
+github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -722,8 +722,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc=
 github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@@ -837,8 +837,8 @@ github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766 h1:wa2wak7
 github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766/go.mod h1:bFATagikCvdPOsmaN1h5VQSbZjTW8bCQ6bjoQEePUq8=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/utils v0.0.58 h1:kk2AkSO84QZc9rDRI8jWA2Iia4uzb4sUcfh4h0xA20I=
-github.com/projectdiscovery/utils v0.0.58/go.mod h1:rsR5Kzjrb+/Yp7JSnEblLk4LfU4zH5Z7wQn8RzaGSdY=
+github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6 h1:60DKG3aueYiy93ZPt78yyZW0N+b7pWbK8Ub1UH6o08I=
+github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6/go.mod h1:vt4oY4rvRWTdkBMhLlAGPbapa/R8pa+xZBYuNZIKJgQ=
 github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k=
 github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

integration_tests/flow/flow-hide-matcher.yaml

@@ -0,0 +1,28 @@
+id: flow-hide-matcher
+
+info:
+  name: Test HTTP Template
+  author: pdteam
+  severity: info
+  description: In flow matcher output of previous step is hidden and only last event matcher output is shown
+
+flow: http(1) && http(2)
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - ok
+
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "Failed event"

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a00473045022100d407a3b848664b4c271abb4462a89a53fa2da6c21fd66011974ac395e2dc041c0220129a752a792337f6efe2e96562989016fe2709820b9583fd933f02be3b9d074f:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a004830460221009db4541aa2af10aae5f39fe6e8789e2717c96ebbdadfdf33114ec0e82ec4da73022100fa98ee6611b606befc139946a169cca717f16ebf71beac97fdde1fe0c7fba774:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4a0a0047304502205ebee72972ea0005ecdbcf7cd676ab861f3a44477a4b85dc1e745b7a628d2d7a022100ec4604673a1d43311ab343005464be5d4ee26b5a1f39206aa841056f3e2057dd:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-snippet.yaml

@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a004830460221009a87b77e770e688bb1ce05e75ac075cdb3f318aad18a6dbc3fc2ec729a8ba5990221009020d69ba3baf47f9d835d4b6bd644a9e4f2d699369acc2a15983f5c270d2e79:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/javascript/net-https.yaml

@@ -0,0 +1,25 @@
+id: net-https
+
+info:
+  name: net-https
+  author: pdteam
+  severity: info
+  description: send and receive https data using net module
+
+
+javascript:
+  - code: |
+      let m = require('nuclei/net');
+      let name=Host+':'+Port;
+      let conn = m.OpenTLS('tcp', name);
+      conn.Send('GET / HTTP/1.1\r\nHost:'+name+'\r\nConnection: close\r\n\r\n');
+      resp = conn.RecvString();
+
+    args:
+      Host: "{{Host}}"
+      Port: "443"
+
+    matchers:
+      - type: word
+        words:
+          - "HTTP/1.1 200 OK"