Merge branch 'dev'

README.md

@@ -126,9 +126,9 @@ TEMPLATES:
    -ntv, -new-templates-version string[]  run new templates added in specific version
    -as, -automatic-scan                   automatic web scan using wappalyzer technology detection to tags mapping
    -t, -templates string[]                list of template or template directory to run (comma-separated, file)
-   -tu, -template-url string[]            list of template urls to run (comma-separated, file)
+   -turl, -template-url string[]          template url or list containing template urls to run (comma-separated, file)
    -w, -workflows string[]                list of workflow or workflow directory to run (comma-separated, file)
-   -wu, -workflow-url string[]            list of workflow urls to run (comma-separated, file)
+   -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content

docs/template-guide/operators/matchers.mdx

@@ -8,7 +8,7 @@ Matchers allow different type of flexible comparisons on protocol responses. The
 
 ### Types
 
-Multiple matchers can be specified in a request. There are basically 6 types of matchers:
+Multiple matchers can be specified in a request. There are basically 7 types of matchers:
 
 | Matcher Type | Part Matched                |
 |--------------|-----------------------------|
@@ -18,6 +18,7 @@ Multiple matchers can be specified in a request. There are basically 6 types of
 | regex        | Part for a protocol         |
 | binary       | Part for a protocol         |
 | dsl          | Part for a protocol         |
+| xpath        | Part for a protocol         |
 
 To match status codes for responses, you can use the following syntax.
 
@@ -57,6 +58,16 @@ matchers:
 
 **Word** and **Regex** matchers can be further configured depending on the needs of the users.
 
+**XPath** matchers use XPath queries to match XML and HTML responses. If the XPath query returns any results, it's considered a match.
+
+```yaml
+matchers:
+  - type: xpath
+    part: body
+    xpath:
+      - "/html/head/title[contains(text(), 'Example Domain')]"
+```
+
 Complex matchers of type **dsl** allows building more elaborate expressions with helper functions. These function allow access to Protocol Response which contains variety of data based on each protocol. See protocol specific documentation to learn about different returned results.
 
 

integration_tests/headless/file-upload-negative.yaml

@@ -0,0 +1,29 @@
+id: file-upload
+# template for testing when file upload is disabled
+info:
+  name: Basic File Upload
+  author: pdteam
+  severity: info
+
+headless:
+  - steps:
+    - action: navigate
+      args:
+        url: "{{BaseURL}}"
+    - action: waitload
+    - action: files
+      args:
+        by: xpath
+        xpath: /html/body/form/input[1]
+        value: headless/file-upload.yaml
+    - action: sleep
+      args:
+        duration: 2
+    - action: click
+      args:
+        by: x
+        xpath: /html/body/form/input[2]
+    matchers:
+      - type: word
+        words:
+          - "Basic File Upload"

integration_tests/headless/headless-local.yaml

@@ -0,0 +1,15 @@
+id: nuclei-headless-local
+
+info:
+  name: Nuclei Headless Local
+  author: pdteam
+  severity: high
+
+headless:
+  - steps: 
+    - action: navigate
+      args:
+        url: "{{BaseURL}}"
+
+    - action: waitload
+

nuclei-jsonschema.json

@@ -342,6 +342,14 @@
           "title": "dsl expressions to match in response",
           "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"
         },
+        "xpath": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "xpath queries to match in response",
+          "description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"
+        },
         "encoding": {
           "enum": [
             "hex"
@@ -371,7 +379,8 @@
         "binary",
         "status",
         "size",
-        "dsl"
+        "dsl",
+        "xpath"
       ],
       "type": "string",
       "title": "type of the matcher",

v2/cmd/integration-test/headless.go

@@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{
 	{Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},
 	{Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}},
 	{Path: "headless/variables.yaml", TestCase: &headlessVariables{}},
+	{Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}},
 	{Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}},
+	{Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}},
 	{Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}},
 }
 
@@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type headlessLocal struct{}
+
+// Execute executes a test case and returns an error if occurred
+// in this testcases local network access is disabled
+func (h *headlessLocal) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte("<html><body></body></html>"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"}
+
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}
+
 type headlessHeaderActions struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type headlessFileUploadNegative struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessFileUploadNegative) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte(`
+		<!doctype html>
+			<body>
+				<form method=post enctype=multipart/form-data>
+				<input type=file name=file>
+				<input type=submit value=Upload>
+				</form>
+			</body>
+		</html>
+		`))
+	})
+	router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, _, err := r.FormFile("file")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		defer file.Close()
+
+		content, err := io.ReadAll(file)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		_, _ = w.Write(content)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+	args := []string{"-t", filePath, "-u", ts.URL, "-headless"}
+
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}

v2/cmd/integration-test/loader.go

@@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
 	if err == nil {
 		return fmt.Errorf("expected error for not allowed remote template list url")
 	}
@@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
@@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}

v2/cmd/nuclei/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/interactsh/pkg/client"
+	"github.com/projectdiscovery/nuclei/v2/internal/installer"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
@@ -136,14 +137,14 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", nil, "list of template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", nil, "list of workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
-		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"),
+		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"templates.nuclei.sh"}, "allowed domain list to load remote templates from"),
 	)
 
 	flagSet.CreateGroup("filters", "Filtering",
@@ -340,6 +341,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
 
+	if options.VerboseVerbose {
+		// hide release notes if silent mode is enabled
+		installer.HideReleaseNotes = false
+	}
+
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}

v2/internal/installer/template.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/charmbracelet/glamour"
 	"github.com/olekukonko/tablewriter"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
@@ -29,6 +30,7 @@ const (
 var (
 	HideProgressBar        = true
 	HideUpdateChangesTable = false
+	HideReleaseNotes       = true
 )
 
 // TemplateUpdateResults contains the results of template update
@@ -105,6 +107,7 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
 	}
+
 	// write templates to disk
 	if err := t.writeTemplatesToDisk(ghrd, dir); err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir)
@@ -313,6 +316,21 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
 		return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index")
 	}
 
+	if !HideReleaseNotes {
+		output := ghrd.Latest.GetBody()
+		// adjust colors for both dark / light terminal themes
+		r, err := glamour.NewTermRenderer(glamour.WithAutoStyle())
+		if err != nil {
+			gologger.Error().Msgf("markdown rendering not supported: %v", err)
+		}
+		if rendered, err := r.Render(output); err == nil {
+			output = rendered
+		} else {
+			gologger.Error().Msg(err.Error())
+		}
+		gologger.Print().Msgf("\n%v\n\n", output)
+	}
+
 	// after installation, create and write checksums to .checksum file
 	return t.writeChecksumFileInDir(dir)
 }

v2/internal/runner/templates.go

@@ -2,7 +2,6 @@ package runner
 
 import (
 	"bytes"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -45,22 +44,19 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 		if hasExtraFlags(r.options) {
 			if r.options.TemplateDisplay {
 				colorize := !r.options.NoColor
-
 				path := tpl.Path
-				tplBody, err := os.ReadFile(path)
+				tplBody, err := store.ReadTemplateFromURI(path, true)
 				if err != nil {
 					gologger.Error().Msgf("Could not read the template %s: %s", path, err)
 					continue
 				}
-
 				if colorize {
 					path = aurora.Cyan(tpl.Path).String()
 					tplBody, err = r.highlightTemplate(&tplBody)
 					if err != nil {
 						gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err)
 						continue
 					}
-
 				}
 				gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody)
 			} else {

v2/pkg/catalog/config/constants.go

@@ -17,7 +17,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v2.9.12`
+	Version = `v2.9.13`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"