diff --git a/README.md b/README.md index 7007930a03..8c770a234a 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,9 @@ TEMPLATES: -ntv, -new-templates-version string[] run new templates added in specific version -as, -automatic-scan automatic web scan using wappalyzer technology detection to tags mapping -t, -templates string[] list of template or template directory to run (comma-separated, file) - -tu, -template-url string[] list of template urls to run (comma-separated, file) + -turl, -template-url string[] template url or list containing template urls to run (comma-separated, file) -w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file) - -wu, -workflow-url string[] list of workflow urls to run (comma-separated, file) + -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -validate validate the passed templates to nuclei -nss, -no-strict-syntax disable strict syntax check on templates -td, -template-display displays the templates content diff --git a/docs/template-guide/operators/matchers.mdx b/docs/template-guide/operators/matchers.mdx index 8a87ac4b3a..4a9da9cea4 100644 --- a/docs/template-guide/operators/matchers.mdx +++ b/docs/template-guide/operators/matchers.mdx @@ -8,7 +8,7 @@ Matchers allow different type of flexible comparisons on protocol responses. The ### Types -Multiple matchers can be specified in a request. There are basically 6 types of matchers: +Multiple matchers can be specified in a request. There are basically 7 types of matchers: | Matcher Type | Part Matched | |--------------|-----------------------------| @@ -18,6 +18,7 @@ Multiple matchers can be specified in a request. There are basically 6 types of | regex | Part for a protocol | | binary | Part for a protocol | | dsl | Part for a protocol | +| xpath | Part for a protocol | To match status codes for responses, you can use the following syntax. @@ -57,6 +58,16 @@ matchers: **Word** and **Regex** matchers can be further configured depending on the needs of the users. +**XPath** matchers use XPath queries to match XML and HTML responses. If the XPath query returns any results, it's considered a match. + +```yaml +matchers: + - type: xpath + part: body + xpath: + - "/html/head/title[contains(text(), 'Example Domain')]" +``` + Complex matchers of type **dsl** allows building more elaborate expressions with helper functions. These function allow access to Protocol Response which contains variety of data based on each protocol. See protocol specific documentation to learn about different returned results. diff --git a/integration_tests/headless/file-upload-negative.yaml b/integration_tests/headless/file-upload-negative.yaml new file mode 100644 index 0000000000..3d8c2bf4a0 --- /dev/null +++ b/integration_tests/headless/file-upload-negative.yaml @@ -0,0 +1,29 @@ +id: file-upload +# template for testing when file upload is disabled +info: + name: Basic File Upload + author: pdteam + severity: info + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + - action: waitload + - action: files + args: + by: xpath + xpath: /html/body/form/input[1] + value: headless/file-upload.yaml + - action: sleep + args: + duration: 2 + - action: click + args: + by: x + xpath: /html/body/form/input[2] + matchers: + - type: word + words: + - "Basic File Upload" \ No newline at end of file diff --git a/integration_tests/headless/headless-local.yaml b/integration_tests/headless/headless-local.yaml new file mode 100644 index 0000000000..385859d0a1 --- /dev/null +++ b/integration_tests/headless/headless-local.yaml @@ -0,0 +1,15 @@ +id: nuclei-headless-local + +info: + name: Nuclei Headless Local + author: pdteam + severity: high + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + + - action: waitload + \ No newline at end of file diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 579db7082f..ff8291eb12 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -342,6 +342,14 @@ "title": "dsl expressions to match in response", "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules" }, + "xpath": { + "items": { + "type": "string" + }, + "type": "array", + "title": "xpath queries to match in response", + "description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules" + }, "encoding": { "enum": [ "hex" @@ -371,7 +379,8 @@ "binary", "status", "size", - "dsl" + "dsl", + "xpath" ], "type": "string", "title": "type of the matcher", diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go index 185cdc87f5..b30c464cc7 100644 --- a/v2/cmd/integration-test/headless.go +++ b/v2/cmd/integration-test/headless.go @@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{ {Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}}, {Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}}, {Path: "headless/variables.yaml", TestCase: &headlessVariables{}}, + {Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}}, {Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}}, + {Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}}, {Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}}, } @@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error { return expectResultsCount(results, 1) } +type headlessLocal struct{} + +// Execute executes a test case and returns an error if occurred +// in this testcases local network access is disabled +func (h *headlessLocal) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"} + + results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + type headlessHeaderActions struct{} // Execute executes a test case and returns an error if occurred @@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type headlessFileUploadNegative struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessFileUploadNegative) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte(` + + +
+ + +
+ + + `)) + }) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, _ = w.Write(content) + }) + ts := httptest.NewServer(router) + defer ts.Close() + args := []string{"-t", filePath, "-u", ts.URL, "-headless"} + + results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index d0ba4aac0b..92b682ce7b 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error { } defer os.Remove("test-config.yaml") - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml") + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml") if err != nil { return err } @@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error { ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list") + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list") if err == nil { return fmt.Errorf("expected error for not allowed remote template list url") } @@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error { } defer os.Remove("test-config.yaml") - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml") + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml") if err != nil { return err } @@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404") + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } @@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404") + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index fa83b520e0..91352b5ddc 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/interactsh/pkg/client" + "github.com/projectdiscovery/nuclei/v2/internal/installer" "github.com/projectdiscovery/nuclei/v2/internal/runner" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -136,14 +137,14 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), - flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", nil, "list of template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), - flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", nil, "list of workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"), flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), - flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"), + flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"templates.nuclei.sh"}, "allowed domain list to load remote templates from"), ) flagSet.CreateGroup("filters", "Filtering", @@ -340,6 +341,11 @@ on extensive configurability, massive extensibility and ease of use.`) gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug) + if options.VerboseVerbose { + // hide release notes if silent mode is enabled + installer.HideReleaseNotes = false + } + if options.LeaveDefaultPorts { http.LeaveDefaultPorts = true } diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go index 22caa5d7fd..3db4924cc8 100644 --- a/v2/internal/installer/template.go +++ b/v2/internal/installer/template.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "github.com/charmbracelet/glamour" "github.com/olekukonko/tablewriter" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" @@ -29,6 +30,7 @@ const ( var ( HideProgressBar = true HideUpdateChangesTable = false + HideReleaseNotes = true ) // TemplateUpdateResults contains the results of template update @@ -105,6 +107,7 @@ func (t *TemplateManager) installTemplatesAt(dir string) error { if err != nil { return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) } + // write templates to disk if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir) @@ -313,6 +316,21 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index") } + if !HideReleaseNotes { + output := ghrd.Latest.GetBody() + // adjust colors for both dark / light terminal themes + r, err := glamour.NewTermRenderer(glamour.WithAutoStyle()) + if err != nil { + gologger.Error().Msgf("markdown rendering not supported: %v", err) + } + if rendered, err := r.Render(output); err == nil { + output = rendered + } else { + gologger.Error().Msg(err.Error()) + } + gologger.Print().Msgf("\n%v\n\n", output) + } + // after installation, create and write checksums to .checksum file return t.writeChecksumFileInDir(dir) } diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 559448732f..f0d9cfe540 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -2,7 +2,6 @@ package runner import ( "bytes" - "os" "path/filepath" "strings" @@ -45,14 +44,12 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { if hasExtraFlags(r.options) { if r.options.TemplateDisplay { colorize := !r.options.NoColor - path := tpl.Path - tplBody, err := os.ReadFile(path) + tplBody, err := store.ReadTemplateFromURI(path, true) if err != nil { gologger.Error().Msgf("Could not read the template %s: %s", path, err) continue } - if colorize { path = aurora.Cyan(tpl.Path).String() tplBody, err = r.highlightTemplate(&tplBody) @@ -60,7 +57,6 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err) continue } - } gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody) } else { diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 830420d168..77a519ad2e 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v2.9.12` + Version = `v2.9.13` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 8535b82334..f633cbd7e0 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -1,8 +1,12 @@ package loader import ( + "fmt" + "io" + "net/url" "os" "sort" + "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -17,6 +21,15 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" +) + +const ( + httpPrefix = "http://" + httpsPrefix = "https://" ) // Config contains the configuration options for the loader @@ -120,6 +133,30 @@ func New(config *Config) (*Store, error) { finalWorkflows: config.Workflows, } + // Do a check to see if we have URLs in templates flag, if so + // we need to processs them separately and remove them from the initial list + var templatesFinal []string + for _, template := range config.Templates { + // TODO: Add and replace this with urlutil.IsURL() helper + if stringsutil.HasPrefixAny(template, httpPrefix, httpsPrefix) { + config.TemplateURLs = append(config.TemplateURLs, template) + } else { + templatesFinal = append(templatesFinal, template) + } + } + + // fix editor paths + remoteTemplates := []string{} + for _, v := range config.TemplateURLs { + if _, err := urlutil.Parse(v); err == nil { + remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v)) + } else { + templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file + } + } + config.TemplateURLs = remoteTemplates + store.finalTemplates = templatesFinal + urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 if urlBasedTemplatesProvided { remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs, config.RemoteTemplateDomainList) @@ -145,6 +182,43 @@ func New(config *Config) (*Store, error) { return store, nil } +func handleTemplatesEditorURLs(input string) string { + parsed, err := url.Parse(input) + if err != nil { + return input + } + if !strings.HasSuffix(parsed.Hostname(), "templates.nuclei.sh") { + return input + } + if strings.HasSuffix(parsed.Path, ".yaml") { + return input + } + parsed.Path = fmt.Sprintf("%s.yaml", parsed.Path) + finalURL := parsed.String() + return finalURL +} + +// ReadTemplateFromURI should only be used for viewing templates +// and should not be used anywhere else like loading and executing templates +// there is no sandbox restriction here +func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) { + if stringsutil.HasPrefixAny(uri, httpPrefix, httpsPrefix) && remote { + uri = handleTemplatesEditorURLs(uri) + remoteTemplates, _, err := getRemoteTemplatesAndWorkflows([]string{uri}, nil, store.config.RemoteTemplateDomainList) + if err != nil || len(remoteTemplates) == 0 { + return nil, errorutil.NewWithErr(err).Msgf("Could not load template %s: got %v", uri, remoteTemplates) + } + resp, err := retryablehttp.Get(remoteTemplates[0]) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(resp.Body) + } else { + return os.ReadFile(uri) + } +} + // Templates returns all the templates in the store func (store *Store) Templates() []*templates.Template { return store.templates diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index c5e0a64324..77bfddaa2b 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -56,7 +56,6 @@ func getRemoteTemplatesAndWorkflows(templateURLs, workflowURLs, remoteTemplateDo } } } - return remoteTemplateList, remoteWorkFlowList, err } diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index aae6f3a941..04b4b1642c 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -5,6 +5,8 @@ import ( "strings" "github.com/Knetic/govaluate" + "github.com/antchfx/htmlquery" + "github.com/antchfx/xmlquery" dslRepo "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" @@ -226,6 +228,88 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { return false } +// MatchXPath matches on a generic map result +func (matcher *Matcher) MatchXPath(corpus string) bool { + if strings.HasPrefix(corpus, " 0 +} + +// MatchXML matches items from XML using XPath selectors +func (matcher *Matcher) MatchXML(corpus string) bool { + doc, err := xmlquery.Parse(strings.NewReader(corpus)) + if err != nil { + return false + } + + matches := 0 + + for _, k := range matcher.XPath { + nodes, err := xmlquery.QueryAll(doc, k) + if err != nil { + continue + } + + // Continue if the xpath doesn't return any nodes + if len(nodes) == 0 { + // If we are in an AND request and a match failed, + // return false as the AND condition fails on any single mismatch. + switch matcher.condition { + case ANDCondition: + return false + case ORCondition: + continue + } + } + + // If the condition was an OR, return on the first match. + if matcher.condition == ORCondition && !matcher.MatchAll { + return true + } + matches = matches + len(nodes) + } + + return matches > 0 +} + // ignoreErr checks if the error is to be ignored or not // Reference: https://github.com/projectdiscovery/nuclei/issues/3950 func (m *Matcher) ignoreErr(err error) bool { diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index db50b11e9c..bc31290edc 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -89,3 +89,123 @@ func TestMatcher_MatchDSL(t *testing.T) { require.True(t, isMatched) } } + +func TestMatcher_MatchXPath_HTML(t *testing.T) { + body := ` + + + Example Domain + + + + + + + +
+

Example Domain

+

This domain is for use in illustrative examples in documents. You may use this + domain in literature without prior coordination or asking for permission.

+

More information...

+
+ + +` + body2 := ` + + + Example Domain + + +

It's test time!

+ + +` + + // single match + m := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"/html/body/div/p[2]/a"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched := m.MatchXPath(body) + require.True(t, isMatched, "Could not match valid XPath") + + isMatched = m.MatchXPath("

aaaaaaaaa") + require.False(t, isMatched, "Could match invalid XPath") + + // OR match + m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "or", XPath: []string{"/html/head/title[contains(text(), 'PATRICAAA')]", "/html/body/div/p[2]/a"}} + err = m.CompileMatchers() + require.Nil(t, err) + + isMatched = m.MatchXPath(body) + require.True(t, isMatched, "Could not match valid multi-XPath with OR condition") + + isMatched = m.MatchXPath(body2) + require.False(t, isMatched, "Could match invalid multi-XPath with OR condition") + + // AND match + m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "and", XPath: []string{"/html/head/title[contains(text(), 'Example Domain')]", "/html/body/div/p[2]/a"}} + err = m.CompileMatchers() + require.Nil(t, err) + + isMatched = m.MatchXPath(body) + require.True(t, isMatched, "Could not match valid multi-XPath with AND condition") + + isMatched = m.MatchXPath(body2) + require.False(t, isMatched, "Could match invalid multi-XPath with AND condition") + + // invalid xpath + m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//a[@a==1]"}} + _ = m.CompileMatchers() + isMatched = m.MatchXPath(body) + require.False(t, isMatched, "Invalid xpath did not return false") +} + +func TestMatcher_MatchXPath_XML(t *testing.T) { + body := `barbaz` + body2 := `baralo` + + // single match + m := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//foo[contains(text(), 'bar')]"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched := m.MatchXPath(body) + require.True(t, isMatched, "Could not match valid XPath") + + isMatched = m.MatchXPath("

aaaaaaaaa

") + require.False(t, isMatched, "Could match invalid XPath") + + // OR match + m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "or", XPath: []string{"/foo[contains(text(), 'PATRICAAA')]", "/parent/child"}} + err = m.CompileMatchers() + require.Nil(t, err) + + isMatched = m.MatchXPath(body) + require.True(t, isMatched, "Could not match valid multi-XPath with OR condition") + + isMatched = m.MatchXPath(body2) + require.False(t, isMatched, "Could match invalid multi-XPath with OR condition") + + // AND match + m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "and", XPath: []string{"/foo[contains(text(), 'bar')]", "/parent/child"}} + err = m.CompileMatchers() + require.Nil(t, err) + + isMatched = m.MatchXPath(body) + require.True(t, isMatched, "Could not match valid multi-XPath with AND condition") + + isMatched = m.MatchXPath(body2) + require.False(t, isMatched, "Could match invalid multi-XPath with AND condition") + + // invalid xpath + m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//a[@a==1]"}} + _ = m.CompileMatchers() + isMatched = m.MatchXPath(body) + require.False(t, isMatched, "Invalid xpath did not return false") + + // invalid xml + isMatched = m.MatchXPath("

not right notvalid") + require.False(t, isMatched, "Invalid xpath did not return false") +} diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 2943c49455..670113d42c 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -94,6 +94,16 @@ type Matcher struct { // []string{"!contains(tolower(all_headers), ''strict-transport-security'')"} DSL []string `yaml:"dsl,omitempty" json:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"` // description: | + // XPath are the xpath queries expressions that will be evaluated against the response part. + // examples: + // - name: XPath Matcher to check a title + // value: > + // []string{"/html/head/title[contains(text(), 'How to Find XPath')]"} + // - name: XPath Matcher for finding links with target="_blank" + // value: > + // []string{"//a[@target="_blank"]"} + XPath []string `yaml:"xpath,omitempty" json:"xpath,omitempty" jsonschema:"title=xpath queries to match in response,description=xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"` + // description: | // Encoding specifies the encoding for the words field if any. // values: // - "hex" diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 0ce6c7e692..9c872192a8 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -25,6 +25,8 @@ const ( SizeMatcher // name:dsl DSLMatcher + // name:xpath + XPathMatcher limit ) @@ -36,6 +38,7 @@ var MatcherTypes = map[MatcherType]string{ RegexMatcher: "regex", BinaryMatcher: "binary", DSLMatcher: "dsl", + XPathMatcher: "xpath", } // GetType returns the type of the matcher diff --git a/v2/pkg/operators/matchers/validate.go b/v2/pkg/operators/matchers/validate.go index 7d347919c1..9e0a7aba81 100644 --- a/v2/pkg/operators/matchers/validate.go +++ b/v2/pkg/operators/matchers/validate.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" + "github.com/antchfx/xpath" sliceutil "github.com/projectdiscovery/utils/slice" "gopkg.in/yaml.v3" ) @@ -38,8 +39,23 @@ func (matcher *Matcher) Validate() error { expectedFields = append(commonExpectedFields, "Binary", "Part", "Encoding", "CaseInsensitive") case RegexMatcher: expectedFields = append(commonExpectedFields, "Regex", "Part", "Encoding", "CaseInsensitive") + case XPathMatcher: + expectedFields = append(commonExpectedFields, "XPath", "Part") } - return checkFields(matcher, matcherMap, expectedFields...) + + if err = checkFields(matcher, matcherMap, expectedFields...); err != nil { + return err + } + + // validate the XPath query + if matcher.matcherType == XPathMatcher { + for _, query := range matcher.XPath { + if _, err = xpath.Compile(query); err != nil { + return err + } + } + } + return nil } func checkFields(m *Matcher, matcherMap map[string]interface{}, expectedFields ...string) error { diff --git a/v2/pkg/operators/matchers/validate_test.go b/v2/pkg/operators/matchers/validate_test.go index e617a9491a..049ac045ff 100644 --- a/v2/pkg/operators/matchers/validate_test.go +++ b/v2/pkg/operators/matchers/validate_test.go @@ -15,4 +15,17 @@ func TestValidate(t *testing.T) { m = &Matcher{matcherType: DSLMatcher, Part: "test"} err = m.Validate() require.NotNil(t, err, "Invalid template was correctly validated") + + m = &Matcher{matcherType: XPathMatcher, XPath: []string{"//q[@id=\"foo\"]"}} + + err = m.Validate() + require.Nil(t, err, "Could not validate correct XPath template") + + m = &Matcher{matcherType: XPathMatcher, Status: []int{123}} + err = m.Validate() + require.NotNil(t, err, "Invalid XPath template was correctly validated") + + m = &Matcher{matcherType: XPathMatcher, XPath: []string{"//a[@a==1]"}} + err = m.Validate() + require.NotNil(t, err, "Invalid XPath query was correctly validated") } diff --git a/v2/pkg/protocols/common/protocolstate/headless.go b/v2/pkg/protocols/common/protocolstate/headless.go new file mode 100644 index 0000000000..86cbb2511b --- /dev/null +++ b/v2/pkg/protocols/common/protocolstate/headless.go @@ -0,0 +1,90 @@ +package protocolstate + +import ( + "strings" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" + "github.com/projectdiscovery/networkpolicy" + errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" + "go.uber.org/multierr" +) + +// initalize state of headless protocol + +var ( + ErrURLDenied = errorutil.NewWithFmt("headless: url %v dropped by rule: %v") + networkPolicy *networkpolicy.NetworkPolicy + allowLocalFileAccess bool +) + +// ValidateNFailRequest validates and fails request +// if the request does not respect the rules, it will be canceled with reason +func ValidateNFailRequest(page *rod.Page, e *proto.FetchRequestPaused) error { + reqURL := e.Request.URL + normalized := strings.ToLower(reqURL) // normalize url to lowercase + normalized = strings.TrimSpace(normalized) // trim leading & trailing whitespaces + if !allowLocalFileAccess && stringsutil.HasPrefixI(normalized, "file:") { + return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "use of file:// protocol disabled use '-lfa' to enable")) + } + // validate potential invalid schemes + // javascript protocol is allowed for xss fuzzing + if HasPrefixAnyI(normalized, "ftp:", "externalfile:", "chrome:", "chrome-extension:") { + return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "protocol blocked by network policy")) + } + if !isValidHost(reqURL) { + return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "address blocked by network policy")) + } + return nil +} + +// FailWithReason fails request with AccessDenied reason +func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error { + m := proto.FetchFailRequest{ + RequestID: e.RequestID, + ErrorReason: proto.NetworkErrorReasonAccessDenied, + } + return m.Call(page) +} + +// InitHeadless initializes headless protocol state +func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) { + allowLocalFileAccess = localFileAccess + if !RestrictLocalNetworkAccess { + return + } + networkPolicy, _ = networkpolicy.New(networkpolicy.Options{ + DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...), + }) +} + +// isValidHost checks if the host is valid (only limited to http/https protocols) +func isValidHost(targetUrl string) bool { + if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") { + return true + } + if networkPolicy == nil { + return true + } + urlx, err := urlutil.Parse(targetUrl) + if err != nil { + // not a valid url + return false + } + targetUrl = urlx.Hostname() + _, ok := networkPolicy.ValidateHost(targetUrl) + return ok +} + +// HasPrefixAnyI checks if the string has any of the prefixes +// TODO: replace with stringsutil.HasPrefixAnyI after implementation +func HasPrefixAnyI(s string, prefixes ...string) bool { + for _, prefix := range prefixes { + if stringsutil.HasPrefixI(s, prefix) { + return true + } + } + return false +} diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index 829c2ce346..d07d1ba1ba 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -22,6 +22,7 @@ func Init(options *types.Options) error { return nil } opts := fastdialer.DefaultOptions + InitHeadless(options.RestrictLocalNetworkAccess, options.AllowLocalFileAccess) switch { case options.SourceIP != "" && options.Interface != "": diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index d2b6be7fa9..d0cad29349 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -42,6 +42,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item))) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), []string{} + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(types.ToString(item))), []string{} } return false, []string{} } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 3b6c2af14f..2a7825dea2 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -30,6 +30,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), []string{} + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(itemStr)), []string{} } return false, []string{} } diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index a566ab37a3..7ce2cd810e 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -13,6 +13,7 @@ import ( "github.com/go-rod/rod/lib/proto" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Page is a single page in an isolated browser instance @@ -40,6 +41,7 @@ type HistoryData struct { type Options struct { Timeout time.Duration CookieReuse bool + Options *types.Options } // Run runs a list of actions by creating a new page in the browser. diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 93cd5018c1..1b3cee291c 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -30,7 +30,8 @@ import ( ) var ( - errinvalidArguments = errors.New("invalid arguments provided") + errinvalidArguments = errorutil.New("invalid arguments provided") + ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access") ) const ( @@ -70,7 +71,11 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var case ActionWaitEvent: err = p.WaitEvent(act, outData) case ActionFilesInput: - err = p.FilesInput(act, outData) + if p.options.Options.AllowLocalFileAccess { + err = p.FilesInput(act, outData) + } else { + err = ErrLFAccessDenied + } case ActionAddHeader: err = p.ActionAddHeader(act, outData) case ActionSetHeader: diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 7930598d21..6b6c50b8f3 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -8,6 +8,7 @@ import ( "net/http/cookiejar" "net/http/httptest" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -20,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless" "github.com/projectdiscovery/nuclei/v2/pkg/types" + stringsutil "github.com/projectdiscovery/utils/strings" ) func TestActionNavigate(t *testing.T) { @@ -36,7 +38,7 @@ func TestActionNavigate(t *testing.T) { actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}} testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { - require.Nil(t, err, "could not run page actions") + require.Nilf(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") }) } @@ -316,6 +318,29 @@ func TestActionFilesInput(t *testing.T) { }) } +// Negative testcase for files input where it should fail +func TestActionFilesInputNegative(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + } + t.Setenv("LOCAL_FILE_ACCESS", "false") + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false") + }) +} + func TestActionWaitLoad(t *testing.T) { response := ` @@ -569,7 +594,10 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle input.CookieJar, err = cookiejar.New(nil) require.Nil(t, err) - extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout}) + lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true) + rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false) + + extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna}}) // allow file access in test assert(page, err, extractedData) if page != nil { @@ -591,3 +619,73 @@ func TestContainsAnyModificationActionType(t *testing.T) { t.Error("Expected true, got false") } } + +func TestBlockedHeadlessURLS(t *testing.T) { + + // run this test from binary since we are changing values + // of global variables + if os.Getenv("TEST_BLOCK_HEADLESS_URLS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestBlockedHeadlessURLS", "-test.v") + cmd.Env = append(cmd.Env, "TEST_BLOCK_HEADLESS_URLS=1") + out, err := cmd.CombinedOutput() + if !strings.Contains(string(out), "PASS\n") || err != nil { + t.Fatalf("%s\n(exit status %v)", string(out), err) + } + return + } + + opts := &types.Options{ + AllowLocalFileAccess: false, + RestrictLocalNetworkAccess: true, + } + err := protocolstate.Init(opts) + require.Nil(t, err, "could not init protocol state") + + browser, err := New(&types.Options{ShowBrowser: false, UseInstalledChrome: testheadless.HeadlessLocal}) + require.Nil(t, err, "could not create browser") + defer browser.Close() + + instance, err := browser.NewInstance() + require.Nil(t, err, "could not create browser instance") + defer instance.Close() + + ts := httptest.NewServer(nil) + defer ts.Close() + + testcases := []string{ + "file:/etc/hosts", + " file:///etc/hosts\r\n", + " fILe:/../../../../etc/hosts", + ts.URL, // local test server + "fTP://example.com:21\r\n", + "ftp://example.com:21", + "chrome://settings", + " chROme://version", + "chrome-extension://version\r", + " chrOme-EXTension://settings", + "view-source:file:/etc/hosts", + } + + for _, testcase := range testcases { + actions := []*Action{ + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": testcase}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + } + + data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test + require.Error(t, err, "expected error for url %s got %v", testcase, data) + require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase) + require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data) + if page != nil { + page.Close() + } + } +} + +func getBoolFromEnv(key string, defaultValue bool) bool { + val := os.Getenv(key) + if val == "" { + return defaultValue + } + return strings.EqualFold(val, "true") +} diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 22c0057fe0..a28176dc19 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -7,6 +7,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) // routingRuleHandler handles proxy rule for actions related to request/response modification @@ -103,6 +104,12 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { // routingRuleHandlerNative handles native proxy rule func (p *Page) routingRuleHandlerNative(e *proto.FetchRequestPaused) error { + // ValidateNFailRequest validates if Local file access is enabled + // and local network access is enables if not it will fail the request + // that don't match the rules + if err := protocolstate.ValidateNFailRequest(p.page, e); err != nil { + return err + } body, _ := FetchGetResponseBody(p.page, e) headers := make(map[string][]string) for _, h := range e.ResponseHeaders { diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index ddf5f672c6..ef552df3a5 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -37,6 +37,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), []string{} + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(itemStr)), []string{} } return false, []string{} } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 500deb18c3..942eb603f5 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -106,6 +106,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p options := &engine.Options{ Timeout: time.Duration(request.options.Options.PageTimeout) * time.Second, CookieReuse: request.CookieReuse, + Options: request.options.Options, } if options.CookieReuse && input.CookieJar == nil { diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 41de04f846..03d75c7a62 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -40,6 +40,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), []string{} + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(item)), []string{} } return false, []string{} } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 38ffcfb1b3..5faf19797d 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -30,6 +30,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), []string{} + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(itemStr)), []string{} } return false, []string{} } diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 007bfe4d0f..bccc8d19b2 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -40,6 +40,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), []string{} + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(item)), []string{} } return false, []string{} } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 4b89af1bee..9e0d2a6ec4 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -205,6 +205,8 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)), nil + case matchers.XPathMatcher: + return matcher.Result(matcher.MatchXPath(item)), []string{} } return false, nil } diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index bd773a9c3c..79cad7e6e8 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -77,6 +77,33 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro return parts, nil } +// RunNucleiArgsAndGetResults returns result,and runtime errors +func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) { + cmd := exec.Command("./nuclei", args...) + if debug { + cmd.Args = append(cmd.Args, "-debug") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + data, err := cmd.Output() + if debug { + fmt.Println(string(data)) + } + if len(data) < 1 && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + } + var parts []string + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} + // RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { cmd := exec.Command("./nuclei")