diff --git a/cmd/integration-test/loader.go b/cmd/integration-test/loader.go index 2c2beb4cd9..8e5ea40ff8 100644 --- a/cmd/integration-test/loader.go +++ b/cmd/integration-test/loader.go @@ -172,7 +172,14 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404") + configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` + err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) + if err != nil { + return err + } + defer os.Remove("test-config.yaml") + + _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404", "-config", "test-config.yaml") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } @@ -188,7 +195,14 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404") + configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` + err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) + if err != nil { + return err + } + defer os.Remove("test-config.yaml") + + _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404", "-config", "test-config.yaml") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } diff --git a/go.mod b/go.mod index b0089222c9..29ea6becc6 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.56 - github.com/projectdiscovery/gologger v1.1.12 + github.com/projectdiscovery/gologger v1.1.13 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 github.com/projectdiscovery/httpx v1.6.3 @@ -95,7 +95,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.8 github.com/projectdiscovery/useragent v0.0.57 - github.com/projectdiscovery/utils v0.1.5 + github.com/projectdiscovery/utils v0.1.6 github.com/projectdiscovery/wappalyzergo v0.1.6 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 901dabeb2b..2ff05a1532 100644 --- a/go.sum +++ b/go.sum @@ -846,8 +846,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= -github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= -github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= +github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E= +github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= @@ -888,8 +888,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM= github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= -github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= -github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4= +github.com/projectdiscovery/utils v0.1.6 h1:fzdN+oCTics3hPPOEV1FWqf/Hpenav5Ml/73aEwpTZc= +github.com/projectdiscovery/utils v0.1.6/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M= github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/integration_tests/protocols/code/pre-condition.yaml b/integration_tests/protocols/code/pre-condition.yaml index 1c44e9579a..2190ffecbe 100644 --- a/integration_tests/protocols/code/pre-condition.yaml +++ b/integration_tests/protocols/code/pre-condition.yaml @@ -23,4 +23,4 @@ code: - type: dsl dsl: - true -# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a0047304502200307590191cb7c766b6c21e5777d345bdddf7adf9d6da8f7d336d585d9ac4a8b022100fd30fb0c7722778eb3d861d60e721d805925b8d8df2b979ef2104c35ec57d5cb:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 9ff947c208..067c183bb7 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a0047304502203fe1d7d52bc2a41886d576a90c82c3be42078baaa4b46e1f3d8519665d6f88b202210081feb82c41150c5b218e226fc4f299ded19f42ba01ef34ba60b0634b4ea6ee12:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index ad69371d05..3521ec902d 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4b0a00483046022100afb5ebff14a40e7f9b679ffc4d93ce7849e33eb398ebb47f2e757cd24831f9dd02210089ffa21b2763e99ebce95dfc5b91e1e62da4ccdc9d2ad5c48584fa350ba335af:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 76d14efb2e..0ccab7a7cd 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4b0a00483046022100939f83e74d43932a5bd792b1fd2c100eec2df60f2b2a8dd56b5c8ef5faa92b17022100f93031b0de373af7d78e623968ea5a2d67c4561ef70e3e6da15aef7e5c853115:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 4837fa7e0e..5e36c1a673 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -21,4 +21,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a00473045022100b8e676ce0c57b60c233a0203539dec20457bbb5f1790d351a5d45405b6668b2602204b1f2fa18e7db099f05329009597ceb2d9b7337562c1a676e8d50ea2f1c6fcbe:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/workflow/code-template-1.yaml b/integration_tests/workflow/code-template-1.yaml index d41a1a6957..81a9773ae0 100644 --- a/integration_tests/workflow/code-template-1.yaml +++ b/integration_tests/workflow/code-template-1.yaml @@ -19,4 +19,4 @@ code: regex: - 'hello from (.*)' group: 1 -# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 490a00463044022050da011362cf08c2cb81e812c7f86d7282afe0562d4bf00d390f1300d19bc910022029e9d305da69e941ac18797645aecb217abde6557f891e141301b48e89a3c0cd:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/workflow/code-template-2.yaml b/integration_tests/workflow/code-template-2.yaml index ddd7d36a4e..1fb9805501 100644 --- a/integration_tests/workflow/code-template-2.yaml +++ b/integration_tests/workflow/code-template-2.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from first" -# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4b0a00483046022100b3b8759c0df028455eb59b1433ac240e5d4604b011bb0c63680bd3cc159ac6f0022100f44aa11b640d11ad0e2902897f4eb51666ab3cd83c31dfd2590f6e43391e39b0:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index 651c3656f6..d58970fec5 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -66,10 +66,10 @@ func (i *HttpInputProvider) Count() int64 { // Iterate over all inputs in order func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) { err := i.format.Parse(i.inputFile, func(request *types.RequestResponse) bool { - return callback(&contextargs.MetaInput{ - ReqResp: request, - Input: request.URL.String(), - }) + metaInput := contextargs.NewMetaInput() + metaInput.ReqResp = request + metaInput.Input = request.URL.String() + return callback(metaInput) }) if err != nil { gologger.Warning().Msgf("Could not parse input file while iterating: %s\n", err) diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index cb48fc7676..b7ad4b4480 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" configTypes "github.com/projectdiscovery/nuclei/v3/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" ) var ( @@ -32,7 +33,7 @@ func IsErrNotImplemented(err error) bool { if err == nil { return false } - if strings.Contains(err.Error(), "provider") && strings.Contains(err.Error(), "does not implement") { + if stringsutil.ContainsAll(err.Error(), "provider", "does not implement") { return true } return false diff --git a/pkg/input/provider/list/hmap.go b/pkg/input/provider/list/hmap.go index f7e3bd14c6..edf3729193 100644 --- a/pkg/input/provider/list/hmap.go +++ b/pkg/input/provider/list/hmap.go @@ -122,7 +122,7 @@ func (i *ListInputProvider) Iterate(callback func(value *contextargs.MetaInput) }) } callbackFunc := func(k, _ []byte) error { - metaInput := &contextargs.MetaInput{} + metaInput := contextargs.NewMetaInput() if err := metaInput.Unmarshal(string(k)); err != nil { return err } @@ -153,14 +153,16 @@ func (i *ListInputProvider) Set(value string) { } return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL) }) - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.setItem(metaInput) return } // Check if input is ip or hostname if iputil.IsIP(urlx.Hostname()) { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.setItem(metaInput) return } @@ -181,7 +183,9 @@ func (i *ListInputProvider) Set(value string) { if ip == "" { continue } - metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip} + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + metaInput.CustomIP = ip i.setItem(metaInput) } return @@ -211,11 +215,13 @@ func (i *ListInputProvider) Set(value string) { } for _, ip := range ips { + metaInput := contextargs.NewMetaInput() if ip != "" { - metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip} + metaInput.Input = URL + metaInput.CustomIP = ip i.setItem(metaInput) } else { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput.Input = URL i.setItem(metaInput) } } @@ -353,7 +359,8 @@ func (i *ListInputProvider) scanInputFromReader(reader io.Reader) { // isExcluded checks if a URL is in the exclusion list func (i *ListInputProvider) isExcluded(URL string) bool { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL key, err := metaInput.MarshalString() if err != nil { gologger.Warning().Msgf("%s\n", err) @@ -378,14 +385,16 @@ func (i *ListInputProvider) Del(value string) { } return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL) }) - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.delItem(metaInput) return } // Check if input is ip or hostname if iputil.IsIP(urlx.Hostname()) { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.delItem(metaInput) return } @@ -406,7 +415,9 @@ func (i *ListInputProvider) Del(value string) { if ip == "" { continue } - metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip} + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + metaInput.CustomIP = ip i.delItem(metaInput) } return @@ -436,11 +447,13 @@ func (i *ListInputProvider) Del(value string) { } for _, ip := range ips { + metaInput := contextargs.NewMetaInput() if ip != "" { - metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip} + metaInput.Input = URL + metaInput.CustomIP = ip i.delItem(metaInput) } else { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput.Input = URL i.delItem(metaInput) } } @@ -514,7 +527,8 @@ func (i *ListInputProvider) addTargets(targets []string) { func (i *ListInputProvider) removeTargets(targets []string) { for _, target := range targets { - metaInput := &contextargs.MetaInput{Input: target} + metaInput := contextargs.NewMetaInput() + metaInput.Input = target i.delItem(metaInput) } } diff --git a/pkg/input/provider/list/hmap_test.go b/pkg/input/provider/list/hmap_test.go index e8a7e8eea8..cd28b247a1 100644 --- a/pkg/input/provider/list/hmap_test.go +++ b/pkg/input/provider/list/hmap_test.go @@ -40,7 +40,7 @@ func Test_expandCIDR(t *testing.T) { // scan got := []string{} input.hostMap.Scan(func(k, _ []byte) error { - var metainput contextargs.MetaInput + metainput := contextargs.NewMetaInput() if err := metainput.Unmarshal(string(k)); err != nil { return err } @@ -141,7 +141,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) { // scan got := []string{} input.hostMap.Scan(func(k, v []byte) error { - var metainput contextargs.MetaInput + metainput := contextargs.NewMetaInput() if err := metainput.Unmarshal(string(k)); err != nil { return err } @@ -184,7 +184,7 @@ func Test_expandASNInputValue(t *testing.T) { // scan the hmap got := []string{} input.hostMap.Scan(func(k, v []byte) error { - var metainput contextargs.MetaInput + metainput := contextargs.NewMetaInput() if err := metainput.Unmarshal(string(k)); err != nil { return err } diff --git a/pkg/input/provider/simple.go b/pkg/input/provider/simple.go index 71b280c2f5..c85f7871b9 100644 --- a/pkg/input/provider/simple.go +++ b/pkg/input/provider/simple.go @@ -43,7 +43,9 @@ func (s *SimpleInputProvider) Iterate(callback func(value *contextargs.MetaInput // Set adds an item to the input provider func (s *SimpleInputProvider) Set(value string) { - s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + s.Inputs = append(s.Inputs, metaInput) } // SetWithProbe adds an item to the input provider with HTTP probing @@ -52,13 +54,17 @@ func (s *SimpleInputProvider) SetWithProbe(value string, probe types.InputLivene if err != nil { return err } - s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: probedValue}) + metaInput := contextargs.NewMetaInput() + metaInput.Input = probedValue + s.Inputs = append(s.Inputs, metaInput) return nil } // SetWithExclusions adds an item to the input provider if it doesn't match any of the exclusions func (s *SimpleInputProvider) SetWithExclusions(value string) error { - s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + s.Inputs = append(s.Inputs, metaInput) return nil } diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go index ed004ae370..af55595498 100644 --- a/pkg/protocols/common/contextargs/contextargs.go +++ b/pkg/protocols/common/contextargs/contextargs.go @@ -50,9 +50,11 @@ func NewWithInput(ctx context.Context, input string) *Context { if err != nil { gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err) } + metaInput := NewMetaInput() + metaInput.Input = input return &Context{ ctx: ctx, - MetaInput: &MetaInput{Input: input}, + MetaInput: metaInput, CookieJar: jar, args: &mapsutil.SyncLockMap[string, interface{}]{ Map: make(map[string]interface{}), diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go index 72eac6247d..b3c0dabe7a 100644 --- a/pkg/protocols/common/contextargs/metainput.go +++ b/pkg/protocols/common/contextargs/metainput.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "strings" + "sync" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -24,6 +25,12 @@ type MetaInput struct { // ReqResp is the raw request for the input ReqResp *types.RequestResponse `json:"raw-request,omitempty"` + + mu *sync.Mutex +} + +func NewMetaInput() *MetaInput { + return &MetaInput{mu: &sync.Mutex{}} } func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) { @@ -135,10 +142,9 @@ func (metaInput *MetaInput) Unmarshal(data string) error { } func (metaInput *MetaInput) Clone() *MetaInput { - input := &MetaInput{ - Input: metaInput.Input, - CustomIP: metaInput.CustomIP, - } + input := NewMetaInput() + input.Input = metaInput.Input + input.CustomIP = metaInput.CustomIP if metaInput.ReqResp != nil { input.ReqResp = metaInput.ReqResp.Clone() } @@ -160,6 +166,9 @@ func (metaInput *MetaInput) GetScanHash(templateId string) string { // there may be some cases where metainput is changed ex: while executing self-contained template etc // but that totally changes the scanID/hash so to avoid that we compute hash only once // and reuse it for all subsequent calls + metaInput.mu.Lock() + defer metaInput.mu.Unlock() + if metaInput.hash == "" { var rawRequest string if metaInput.ReqResp != nil { diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 38a7ff3290..1630e97c7d 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/utils/errkit" + stringsutil "github.com/projectdiscovery/utils/strings" ) // CacheInterface defines the signature of the hosterrorscache so that @@ -200,7 +201,7 @@ func (c *Cache) checkError(err error) bool { errX := errkit.FromError(err) tmp := errX.Cause() cause := tmp.Error() - if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") { + if stringsutil.ContainsAll(cause, "ReadStatusLine:", "read: connection reset by peer") { // this is a FP and should not be counted as a host error // because server closes connection when it reads corrupted bytes which we send via rawhttp return false diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go index f24e421b86..92e42fe96b 100644 --- a/pkg/protocols/http/httpclientpool/options.go +++ b/pkg/protocols/http/httpclientpool/options.go @@ -7,6 +7,7 @@ import ( "time" "github.com/projectdiscovery/rawhttp" + stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ) @@ -36,7 +37,7 @@ func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Res resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options) if err != nil { cause := err.Error() - if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") { + if stringsutil.ContainsAll(cause, "ReadStatusLine: ", "read: connection reset by peer") { // this error is caused when rawhttp client sends a corrupted or malformed request packet to server // some servers may attempt gracefully shutdown but most will just abruptly close the connection which results // in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 69fbfde4c0..b8f59ac7d4 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -77,6 +77,9 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { s.events = append(s.events, e) } + e.RLock() + defer e.RUnlock() + s.results = append(s.results, e.Results...) } diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go index 054a0bc41e..7cf6f8e21a 100644 --- a/pkg/templates/preprocessors.go +++ b/pkg/templates/preprocessors.go @@ -5,6 +5,7 @@ import ( "regexp" "strings" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/segmentio/ksuid" ) @@ -48,7 +49,7 @@ func (r *randStrPreprocessor) ProcessNReturnData(data []byte) ([]byte, map[strin continue } value := expression[1] - if strings.Contains(value, "(") || strings.Contains(value, ")") { + if stringsutil.ContainsAny(value, "(", ")") { continue } diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go index 8b5a71b1ab..c8a67e41cb 100644 --- a/pkg/testutils/integration.go +++ b/pkg/testutils/integration.go @@ -14,6 +14,7 @@ import ( "github.com/gobwas/ws" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/utils/conversion" ) // ExtraArgs @@ -70,20 +71,26 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ( } else { cmd.Args = append(cmd.Args, "-silent") } - data, err := cmd.Output() - if debug { - fmt.Println(string(data)) + output, err := cmd.Output() + var data string + if len(output) > 0 { + data = strings.TrimSpace(conversion.String(output)) } - if len(data) < 1 && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + if debug { + fmt.Println(data) } var parts []string - items := strings.Split(string(data), "\n") + items := strings.Split(data, "\n") for _, i := range items { if i != "" { parts = append(parts, i) } } + + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), data) + } + return parts, nil } @@ -98,20 +105,25 @@ func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error } else { cmd.Args = append(cmd.Args, "-silent") } - data, err := cmd.Output() - if debug { - fmt.Println(string(data)) + output, err := cmd.Output() + var data string + if len(output) > 0 { + data = strings.TrimSpace(conversion.String(output)) } - if len(data) < 1 && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + if debug { + fmt.Println(data) } var parts []string - items := strings.Split(string(data), "\n") + items := strings.Split(data, "\n") for _, i := range items { if i != "" { parts = append(parts, i) } } + + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), data) + } return parts, nil } @@ -127,12 +139,16 @@ func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]str cmd.Args = append(cmd.Args, "-allow-local-file-access") cmd.Args = append(cmd.Args, "-nc") // disable color cmd.Env = append(cmd.Env, ExtraEnvVars...) - data, err := cmd.CombinedOutput() + dataOutput, err := cmd.CombinedOutput() if debug { - fmt.Println(string(data)) + fmt.Println(string(dataOutput)) + } + var data string + if len(dataOutput) > 0 { + data = strings.TrimSpace(conversion.String(dataOutput)) } results := []string{} - for _, v := range strings.Split(string(data), "\n") { + for _, v := range strings.Split(data, "\n") { line := strings.TrimSpace(v) switch { case strings.HasPrefix(line, "[ERR]"): @@ -164,20 +180,24 @@ func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string } else { cmd.Args = append(cmd.Args, "-silent") } - data, err := cmd.Output() - if debug { - fmt.Println(string(data)) + dataOutput, err := cmd.Output() + var data string + if len(dataOutput) > 0 { + data = strings.TrimSpace(conversion.String(dataOutput)) } - if len(data) < 1 && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + if debug { + fmt.Println(data) } var parts []string - items := strings.Split(string(data), "\n") + items := strings.Split(data, "\n") for _, i := range items { if i != "" { parts = append(parts, i) } } + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), data) + } return parts, nil }