From e146c89930977c7835d1cd8df2a78ba8c39d2d65 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Wed, 23 Aug 2023 18:22:53 +0530
Subject: [PATCH 1/6] render nuclei-templates release (#4082)

---
 v2/cmd/nuclei/main.go             |  6 ++++++
 v2/internal/installer/template.go | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index fa83b520e0..8edb9872bb 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/interactsh/pkg/client"
+	"github.com/projectdiscovery/nuclei/v2/internal/installer"
 	"github.com/projectdiscovery/nuclei/v2/internal/runner"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
@@ -340,6 +341,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
 
+	if options.Silent {
+		// hide release notes if silent mode is enabled
+		installer.HideReleaseNotes = true
+	}
+
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}
diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go
index 22caa5d7fd..1f514bbfbc 100644
--- a/v2/internal/installer/template.go
+++ b/v2/internal/installer/template.go
@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/charmbracelet/glamour"
 	"github.com/olekukonko/tablewriter"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
@@ -29,6 +30,7 @@ const (
 var (
 	HideProgressBar        = true
 	HideUpdateChangesTable = false
+	HideReleaseNotes       = false
 )
 
 // TemplateUpdateResults contains the results of template update
@@ -105,6 +107,7 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
 	}
+
 	// write templates to disk
 	if err := t.writeTemplatesToDisk(ghrd, dir); err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir)
@@ -313,6 +316,21 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo
 		return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index")
 	}
 
+	if !HideReleaseNotes {
+		output := ghrd.Latest.GetBody()
+		// adjust colors for both dark / light terminal themes
+		r, err := glamour.NewTermRenderer(glamour.WithAutoStyle())
+		if err != nil {
+			gologger.Error().Msgf("markdown rendering not supported: %v", err)
+		}
+		if rendered, err := r.Render(output); err == nil {
+			output = rendered
+		} else {
+			gologger.Error().Msg(err.Error())
+		}
+		gologger.Print().Msgf("\n%v\n\n", output)
+	}
+
 	// after installation, create and write checksums to .checksum file
 	return t.writeChecksumFileInDir(dir)
 }

From d3928e080d819f25306bfbaecc399f03516db5aa Mon Sep 17 00:00:00 2001
From: Sandeep Singh <sandeep@projectdiscovery.io>
Date: Fri, 25 Aug 2023 18:30:46 +0530
Subject: [PATCH 2/6] optional file read in headless protocol (#4055)

* use -lfa and -lna in headless

* fix lna in headless

* misc update

* fix nil pointer dereference in test

* fix lint & unit test

* use urlutil

* headless protocol scheme improvements

* add unit and integration tests

* run unit test from binary

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 .../headless/file-upload-negative.yaml        |  29 +++++
 .../headless/headless-local.yaml              |  15 +++
 v2/cmd/integration-test/headless.go           |  68 ++++++++++++
 .../common/protocolstate/headless.go          |  90 ++++++++++++++++
 .../protocols/common/protocolstate/state.go   |   1 +
 v2/pkg/protocols/headless/engine/page.go      |   2 +
 .../protocols/headless/engine/page_actions.go |   9 +-
 .../headless/engine/page_actions_test.go      | 102 +++++++++++++++++-
 v2/pkg/protocols/headless/engine/rules.go     |   7 ++
 v2/pkg/protocols/headless/request.go          |   1 +
 v2/pkg/testutils/integration.go               |  27 +++++
 11 files changed, 347 insertions(+), 4 deletions(-)
 create mode 100644 integration_tests/headless/file-upload-negative.yaml
 create mode 100644 integration_tests/headless/headless-local.yaml
 create mode 100644 v2/pkg/protocols/common/protocolstate/headless.go

diff --git a/integration_tests/headless/file-upload-negative.yaml b/integration_tests/headless/file-upload-negative.yaml
new file mode 100644
index 0000000000..3d8c2bf4a0
--- /dev/null
+++ b/integration_tests/headless/file-upload-negative.yaml
@@ -0,0 +1,29 @@
+id: file-upload
+# template for testing when file upload is disabled
+info:
+  name: Basic File Upload
+  author: pdteam
+  severity: info
+
+headless:
+  - steps:
+    - action: navigate
+      args:
+        url: "{{BaseURL}}"
+    - action: waitload
+    - action: files
+      args:
+        by: xpath
+        xpath: /html/body/form/input[1]
+        value: headless/file-upload.yaml
+    - action: sleep
+      args:
+        duration: 2
+    - action: click
+      args:
+        by: x
+        xpath: /html/body/form/input[2]
+    matchers:
+      - type: word
+        words:
+          - "Basic File Upload"
\ No newline at end of file
diff --git a/integration_tests/headless/headless-local.yaml b/integration_tests/headless/headless-local.yaml
new file mode 100644
index 0000000000..385859d0a1
--- /dev/null
+++ b/integration_tests/headless/headless-local.yaml
@@ -0,0 +1,15 @@
+id: nuclei-headless-local
+
+info:
+  name: Nuclei Headless Local
+  author: pdteam
+  severity: high
+
+headless:
+  - steps: 
+    - action: navigate
+      args:
+        url: "{{BaseURL}}"
+
+    - action: waitload
+    
\ No newline at end of file
diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go
index 185cdc87f5..b30c464cc7 100644
--- a/v2/cmd/integration-test/headless.go
+++ b/v2/cmd/integration-test/headless.go
@@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{
 	{Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},
 	{Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}},
 	{Path: "headless/variables.yaml", TestCase: &headlessVariables{}},
+	{Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}},
 	{Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}},
+	{Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}},
 	{Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}},
 }
 
@@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type headlessLocal struct{}
+
+// Execute executes a test case and returns an error if occurred
+// in this testcases local network access is disabled
+func (h *headlessLocal) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte("<html><body></body></html>"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"}
+
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}
+
 type headlessHeaderActions struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type headlessFileUploadNegative struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessFileUploadNegative) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte(`
+		<!doctype html>
+			<body>
+				<form method=post enctype=multipart/form-data>
+				<input type=file name=file>
+				<input type=submit value=Upload>
+				</form>
+			</body>
+		</html>
+		`))
+	})
+	router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, _, err := r.FormFile("file")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		defer file.Close()
+
+		content, err := io.ReadAll(file)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		_, _ = w.Write(content)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+	args := []string{"-t", filePath, "-u", ts.URL, "-headless"}
+
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}
diff --git a/v2/pkg/protocols/common/protocolstate/headless.go b/v2/pkg/protocols/common/protocolstate/headless.go
new file mode 100644
index 0000000000..86cbb2511b
--- /dev/null
+++ b/v2/pkg/protocols/common/protocolstate/headless.go
@@ -0,0 +1,90 @@
+package protocolstate
+
+import (
+	"strings"
+
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
+	"github.com/projectdiscovery/networkpolicy"
+	errorutil "github.com/projectdiscovery/utils/errors"
+	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
+	"go.uber.org/multierr"
+)
+
+// initalize state of headless protocol
+
+var (
+	ErrURLDenied         = errorutil.NewWithFmt("headless: url %v dropped by rule: %v")
+	networkPolicy        *networkpolicy.NetworkPolicy
+	allowLocalFileAccess bool
+)
+
+// ValidateNFailRequest validates and fails request
+// if the request does not respect the rules, it will be canceled with reason
+func ValidateNFailRequest(page *rod.Page, e *proto.FetchRequestPaused) error {
+	reqURL := e.Request.URL
+	normalized := strings.ToLower(reqURL)      // normalize url to lowercase
+	normalized = strings.TrimSpace(normalized) // trim leading & trailing whitespaces
+	if !allowLocalFileAccess && stringsutil.HasPrefixI(normalized, "file:") {
+		return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "use of file:// protocol disabled use '-lfa' to enable"))
+	}
+	// validate potential invalid schemes
+	// javascript protocol is allowed for xss fuzzing
+	if HasPrefixAnyI(normalized, "ftp:", "externalfile:", "chrome:", "chrome-extension:") {
+		return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "protocol blocked by network policy"))
+	}
+	if !isValidHost(reqURL) {
+		return multierr.Combine(FailWithReason(page, e), ErrURLDenied.Msgf(reqURL, "address blocked by network policy"))
+	}
+	return nil
+}
+
+// FailWithReason fails request with AccessDenied reason
+func FailWithReason(page *rod.Page, e *proto.FetchRequestPaused) error {
+	m := proto.FetchFailRequest{
+		RequestID:   e.RequestID,
+		ErrorReason: proto.NetworkErrorReasonAccessDenied,
+	}
+	return m.Call(page)
+}
+
+// InitHeadless initializes headless protocol state
+func InitHeadless(RestrictLocalNetworkAccess bool, localFileAccess bool) {
+	allowLocalFileAccess = localFileAccess
+	if !RestrictLocalNetworkAccess {
+		return
+	}
+	networkPolicy, _ = networkpolicy.New(networkpolicy.Options{
+		DenyList: append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...),
+	})
+}
+
+// isValidHost checks if the host is valid (only limited to http/https protocols)
+func isValidHost(targetUrl string) bool {
+	if !stringsutil.HasPrefixAny(targetUrl, "http:", "https:") {
+		return true
+	}
+	if networkPolicy == nil {
+		return true
+	}
+	urlx, err := urlutil.Parse(targetUrl)
+	if err != nil {
+		// not a valid url
+		return false
+	}
+	targetUrl = urlx.Hostname()
+	_, ok := networkPolicy.ValidateHost(targetUrl)
+	return ok
+}
+
+// HasPrefixAnyI checks if the string has any of the prefixes
+// TODO: replace with stringsutil.HasPrefixAnyI after implementation
+func HasPrefixAnyI(s string, prefixes ...string) bool {
+	for _, prefix := range prefixes {
+		if stringsutil.HasPrefixI(s, prefix) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go
index 829c2ce346..d07d1ba1ba 100644
--- a/v2/pkg/protocols/common/protocolstate/state.go
+++ b/v2/pkg/protocols/common/protocolstate/state.go
@@ -22,6 +22,7 @@ func Init(options *types.Options) error {
 		return nil
 	}
 	opts := fastdialer.DefaultOptions
+	InitHeadless(options.RestrictLocalNetworkAccess, options.AllowLocalFileAccess)
 
 	switch {
 	case options.SourceIP != "" && options.Interface != "":
diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go
index a566ab37a3..7ce2cd810e 100644
--- a/v2/pkg/protocols/headless/engine/page.go
+++ b/v2/pkg/protocols/headless/engine/page.go
@@ -13,6 +13,7 @@ import (
 	"github.com/go-rod/rod/lib/proto"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
 // Page is a single page in an isolated browser instance
@@ -40,6 +41,7 @@ type HistoryData struct {
 type Options struct {
 	Timeout     time.Duration
 	CookieReuse bool
+	Options     *types.Options
 }
 
 // Run runs a list of actions by creating a new page in the browser.
diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go
index 93cd5018c1..1b3cee291c 100644
--- a/v2/pkg/protocols/headless/engine/page_actions.go
+++ b/v2/pkg/protocols/headless/engine/page_actions.go
@@ -30,7 +30,8 @@ import (
 )
 
 var (
-	errinvalidArguments = errors.New("invalid arguments provided")
+	errinvalidArguments = errorutil.New("invalid arguments provided")
+	ErrLFAccessDenied   = errorutil.New("Use -allow-local-file-access flag to enable local file access")
 )
 
 const (
@@ -70,7 +71,11 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
 		case ActionWaitEvent:
 			err = p.WaitEvent(act, outData)
 		case ActionFilesInput:
-			err = p.FilesInput(act, outData)
+			if p.options.Options.AllowLocalFileAccess {
+				err = p.FilesInput(act, outData)
+			} else {
+				err = ErrLFAccessDenied
+			}
 		case ActionAddHeader:
 			err = p.ActionAddHeader(act, outData)
 		case ActionSetHeader:
diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go
index 7930598d21..6b6c50b8f3 100644
--- a/v2/pkg/protocols/headless/engine/page_actions_test.go
+++ b/v2/pkg/protocols/headless/engine/page_actions_test.go
@@ -8,6 +8,7 @@ import (
 	"net/http/cookiejar"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -20,6 +21,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils/testheadless"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 func TestActionNavigate(t *testing.T) {
@@ -36,7 +38,7 @@ func TestActionNavigate(t *testing.T) {
 	actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}}
 
 	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
-		require.Nil(t, err, "could not run page actions")
+		require.Nilf(t, err, "could not run page actions")
 		require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
 	})
 }
@@ -316,6 +318,29 @@ func TestActionFilesInput(t *testing.T) {
 	})
 }
 
+// Negative testcase for files input where it should fail
+func TestActionFilesInputNegative(t *testing.T) {
+	response := `
+		<html>
+			<head>
+				<title>Nuclei Test Page</title>
+			</head>
+			<body>Nuclei Test Page</body>
+			<input type="file">
+		</html>`
+
+	actions := []*Action{
+		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+		{ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
+	}
+	t.Setenv("LOCAL_FILE_ACCESS", "false")
+
+	testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+		require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false")
+	})
+}
+
 func TestActionWaitLoad(t *testing.T) {
 	response := `
 		<html>
@@ -569,7 +594,10 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
 	input.CookieJar, err = cookiejar.New(nil)
 	require.Nil(t, err)
 
-	extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout})
+	lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true)
+	rna := getBoolFromEnv("RESTRICED_LOCAL_NETWORK_ACCESS", false)
+
+	extractedData, page, err := instance.Run(input, actions, nil, &Options{Timeout: timeout, Options: &types.Options{AllowLocalFileAccess: lfa, RestrictLocalNetworkAccess: rna}}) // allow file access in test
 	assert(page, err, extractedData)
 
 	if page != nil {
@@ -591,3 +619,73 @@ func TestContainsAnyModificationActionType(t *testing.T) {
 		t.Error("Expected true, got false")
 	}
 }
+
+func TestBlockedHeadlessURLS(t *testing.T) {
+
+	// run this test from binary since we are changing values
+	// of global variables
+	if os.Getenv("TEST_BLOCK_HEADLESS_URLS") != "1" {
+		cmd := exec.Command(os.Args[0], "-test.run=TestBlockedHeadlessURLS", "-test.v")
+		cmd.Env = append(cmd.Env, "TEST_BLOCK_HEADLESS_URLS=1")
+		out, err := cmd.CombinedOutput()
+		if !strings.Contains(string(out), "PASS\n") || err != nil {
+			t.Fatalf("%s\n(exit status %v)", string(out), err)
+		}
+		return
+	}
+
+	opts := &types.Options{
+		AllowLocalFileAccess:       false,
+		RestrictLocalNetworkAccess: true,
+	}
+	err := protocolstate.Init(opts)
+	require.Nil(t, err, "could not init protocol state")
+
+	browser, err := New(&types.Options{ShowBrowser: false, UseInstalledChrome: testheadless.HeadlessLocal})
+	require.Nil(t, err, "could not create browser")
+	defer browser.Close()
+
+	instance, err := browser.NewInstance()
+	require.Nil(t, err, "could not create browser instance")
+	defer instance.Close()
+
+	ts := httptest.NewServer(nil)
+	defer ts.Close()
+
+	testcases := []string{
+		"file:/etc/hosts",
+		" file:///etc/hosts\r\n",
+		"	fILe:/../../../../etc/hosts",
+		ts.URL, // local test server
+		"fTP://example.com:21\r\n",
+		"ftp://example.com:21",
+		"chrome://settings",
+		"	chROme://version",
+		"chrome-extension://version\r",
+		"	chrOme-EXTension://settings",
+		"view-source:file:/etc/hosts",
+	}
+
+	for _, testcase := range testcases {
+		actions := []*Action{
+			{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": testcase}},
+			{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+		}
+
+		data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
+		require.Error(t, err, "expected error for url %s got %v", testcase, data)
+		require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase)
+		require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data)
+		if page != nil {
+			page.Close()
+		}
+	}
+}
+
+func getBoolFromEnv(key string, defaultValue bool) bool {
+	val := os.Getenv(key)
+	if val == "" {
+		return defaultValue
+	}
+	return strings.EqualFold(val, "true")
+}
diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go
index 22c0057fe0..a28176dc19 100644
--- a/v2/pkg/protocols/headless/engine/rules.go
+++ b/v2/pkg/protocols/headless/engine/rules.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/proto"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 )
 
 // routingRuleHandler handles proxy rule for actions related to request/response modification
@@ -103,6 +104,12 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
 
 // routingRuleHandlerNative handles native proxy rule
 func (p *Page) routingRuleHandlerNative(e *proto.FetchRequestPaused) error {
+	// ValidateNFailRequest validates if Local file access is enabled
+	// and local network access is enables if not it will fail the request
+	// that don't match the rules
+	if err := protocolstate.ValidateNFailRequest(p.page, e); err != nil {
+		return err
+	}
 	body, _ := FetchGetResponseBody(p.page, e)
 	headers := make(map[string][]string)
 	for _, h := range e.ResponseHeaders {
diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go
index 500deb18c3..942eb603f5 100644
--- a/v2/pkg/protocols/headless/request.go
+++ b/v2/pkg/protocols/headless/request.go
@@ -106,6 +106,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
 	options := &engine.Options{
 		Timeout:     time.Duration(request.options.Options.PageTimeout) * time.Second,
 		CookieReuse: request.CookieReuse,
+		Options:     request.options.Options,
 	}
 
 	if options.CookieReuse && input.CookieJar == nil {
diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go
index bd773a9c3c..79cad7e6e8 100644
--- a/v2/pkg/testutils/integration.go
+++ b/v2/pkg/testutils/integration.go
@@ -77,6 +77,33 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
 	return parts, nil
 }
 
+// RunNucleiArgsAndGetResults returns result,and runtime errors
+func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei", args...)
+	if debug {
+		cmd.Args = append(cmd.Args, "-debug")
+		cmd.Stderr = os.Stderr
+		fmt.Println(cmd.String())
+	} else {
+		cmd.Args = append(cmd.Args, "-silent")
+	}
+	data, err := cmd.Output()
+	if debug {
+		fmt.Println(string(data))
+	}
+	if len(data) < 1 && err != nil {
+		return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
+	}
+	var parts []string
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
 // RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL)
 func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) {
 	cmd := exec.Command("./nuclei")

From f520d7e843259830f3de5d6958efe8d5ebe6de15 Mon Sep 17 00:00:00 2001
From: DoI <5291556+denandz@users.noreply.github.com>
Date: Sat, 26 Aug 2023 05:11:51 +1200
Subject: [PATCH 3/6] XPath matcher support (#4087)

* Added xpath response matching support

* Add validation for user-supplied XPath

* xpath matcher comment fix

* Added XPath matched documentation

* minor changes: remove warnings

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 docs/template-guide/operators/matchers.mdx  |  13 ++-
 v2/pkg/operators/matchers/match.go          |  84 ++++++++++++++
 v2/pkg/operators/matchers/match_test.go     | 120 ++++++++++++++++++++
 v2/pkg/operators/matchers/matchers.go       |  10 ++
 v2/pkg/operators/matchers/matchers_types.go |   3 +
 v2/pkg/operators/matchers/validate.go       |  18 ++-
 v2/pkg/operators/matchers/validate_test.go  |  13 +++
 v2/pkg/protocols/dns/operators.go           |   2 +
 v2/pkg/protocols/file/operators.go          |   2 +
 v2/pkg/protocols/headless/operators.go      |   2 +
 v2/pkg/protocols/http/operators.go          |   2 +
 v2/pkg/protocols/network/operators.go       |   2 +
 v2/pkg/protocols/offlinehttp/operators.go   |   2 +
 v2/pkg/protocols/protocols.go               |   2 +
 14 files changed, 273 insertions(+), 2 deletions(-)

diff --git a/docs/template-guide/operators/matchers.mdx b/docs/template-guide/operators/matchers.mdx
index 8a87ac4b3a..4a9da9cea4 100644
--- a/docs/template-guide/operators/matchers.mdx
+++ b/docs/template-guide/operators/matchers.mdx
@@ -8,7 +8,7 @@ Matchers allow different type of flexible comparisons on protocol responses. The
 
 ### Types
 
-Multiple matchers can be specified in a request. There are basically 6 types of matchers:
+Multiple matchers can be specified in a request. There are basically 7 types of matchers:
 
 | Matcher Type | Part Matched                |
 |--------------|-----------------------------|
@@ -18,6 +18,7 @@ Multiple matchers can be specified in a request. There are basically 6 types of
 | regex        | Part for a protocol         |
 | binary       | Part for a protocol         |
 | dsl          | Part for a protocol         |
+| xpath        | Part for a protocol         |
 
 To match status codes for responses, you can use the following syntax.
 
@@ -57,6 +58,16 @@ matchers:
 
 **Word** and **Regex** matchers can be further configured depending on the needs of the users.
 
+**XPath** matchers use XPath queries to match XML and HTML responses. If the XPath query returns any results, it's considered a match.
+
+```yaml
+matchers:
+  - type: xpath
+    part: body
+    xpath:
+      - "/html/head/title[contains(text(), 'Example Domain')]"
+```
+
 Complex matchers of type **dsl** allows building more elaborate expressions with helper functions. These function allow access to Protocol Response which contains variety of data based on each protocol. See protocol specific documentation to learn about different returned results.
 
 
diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go
index aae6f3a941..04b4b1642c 100644
--- a/v2/pkg/operators/matchers/match.go
+++ b/v2/pkg/operators/matchers/match.go
@@ -5,6 +5,8 @@ import (
 	"strings"
 
 	"github.com/Knetic/govaluate"
+	"github.com/antchfx/htmlquery"
+	"github.com/antchfx/xmlquery"
 
 	dslRepo "github.com/projectdiscovery/dsl"
 	"github.com/projectdiscovery/gologger"
@@ -226,6 +228,88 @@ func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
 	return false
 }
 
+// MatchXPath matches on a generic map result
+func (matcher *Matcher) MatchXPath(corpus string) bool {
+	if strings.HasPrefix(corpus, "<?xml") {
+		return matcher.MatchXML(corpus)
+	}
+	return matcher.MatchHTML(corpus)
+}
+
+// MatchHTML matches items from HTML using XPath selectors
+func (matcher *Matcher) MatchHTML(corpus string) bool {
+	doc, err := htmlquery.Parse(strings.NewReader(corpus))
+	if err != nil {
+		return false
+	}
+
+	matches := 0
+
+	for _, k := range matcher.XPath {
+		nodes, err := htmlquery.QueryAll(doc, k)
+		if err != nil {
+			continue
+		}
+
+		// Continue if the xpath doesn't return any nodes
+		if len(nodes) == 0 {
+			// If we are in an AND request and a match failed,
+			// return false as the AND condition fails on any single mismatch.
+			switch matcher.condition {
+			case ANDCondition:
+				return false
+			case ORCondition:
+				continue
+			}
+		}
+
+		// If the condition was an OR, return on the first match.
+		if matcher.condition == ORCondition && !matcher.MatchAll {
+			return true
+		}
+
+		matches = matches + len(nodes)
+	}
+	return matches > 0
+}
+
+// MatchXML matches items from XML using XPath selectors
+func (matcher *Matcher) MatchXML(corpus string) bool {
+	doc, err := xmlquery.Parse(strings.NewReader(corpus))
+	if err != nil {
+		return false
+	}
+
+	matches := 0
+
+	for _, k := range matcher.XPath {
+		nodes, err := xmlquery.QueryAll(doc, k)
+		if err != nil {
+			continue
+		}
+
+		// Continue if the xpath doesn't return any nodes
+		if len(nodes) == 0 {
+			// If we are in an AND request and a match failed,
+			// return false as the AND condition fails on any single mismatch.
+			switch matcher.condition {
+			case ANDCondition:
+				return false
+			case ORCondition:
+				continue
+			}
+		}
+
+		// If the condition was an OR, return on the first match.
+		if matcher.condition == ORCondition && !matcher.MatchAll {
+			return true
+		}
+		matches = matches + len(nodes)
+	}
+
+	return matches > 0
+}
+
 // ignoreErr checks if the error is to be ignored or not
 // Reference: https://github.com/projectdiscovery/nuclei/issues/3950
 func (m *Matcher) ignoreErr(err error) bool {
diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go
index db50b11e9c..bc31290edc 100644
--- a/v2/pkg/operators/matchers/match_test.go
+++ b/v2/pkg/operators/matchers/match_test.go
@@ -89,3 +89,123 @@ func TestMatcher_MatchDSL(t *testing.T) {
 		require.True(t, isMatched)
 	}
 }
+
+func TestMatcher_MatchXPath_HTML(t *testing.T) {
+	body := `<!doctype html>
+<html>
+<head>
+    <title>Example Domain</title>
+
+    <meta charset="utf-8" />
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+</head>
+
+<body>
+<div>
+    <h1>Example Domain</h1>
+    <p>This domain is for use in illustrative examples in documents. You may use this
+    domain in literature without prior coordination or asking for permission.</p>
+    <p><a href="https://www.iana.org/domains/example">More information...</a></p>
+</div>
+</body>
+</html>
+`
+	body2 := `<!doctype html>
+<html>
+<head>
+    <title>Example Domain</title>
+</head>
+<body>
+<h1> It's test time! </h1>
+</body>
+</html>
+`
+
+	// single match
+	m := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"/html/body/div/p[2]/a"}}
+	err := m.CompileMatchers()
+	require.Nil(t, err)
+
+	isMatched := m.MatchXPath(body)
+	require.True(t, isMatched, "Could not match valid XPath")
+
+	isMatched = m.MatchXPath("<h1>aaaaaaaaa")
+	require.False(t, isMatched, "Could match invalid XPath")
+
+	// OR match
+	m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "or", XPath: []string{"/html/head/title[contains(text(), 'PATRICAAA')]", "/html/body/div/p[2]/a"}}
+	err = m.CompileMatchers()
+	require.Nil(t, err)
+
+	isMatched = m.MatchXPath(body)
+	require.True(t, isMatched, "Could not match valid multi-XPath with OR condition")
+
+	isMatched = m.MatchXPath(body2)
+	require.False(t, isMatched, "Could match invalid multi-XPath with OR condition")
+
+	// AND match
+	m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "and", XPath: []string{"/html/head/title[contains(text(), 'Example Domain')]", "/html/body/div/p[2]/a"}}
+	err = m.CompileMatchers()
+	require.Nil(t, err)
+
+	isMatched = m.MatchXPath(body)
+	require.True(t, isMatched, "Could not match valid multi-XPath with AND condition")
+
+	isMatched = m.MatchXPath(body2)
+	require.False(t, isMatched, "Could match invalid multi-XPath with AND condition")
+
+	// invalid xpath
+	m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//a[@a==1]"}}
+	_ = m.CompileMatchers()
+	isMatched = m.MatchXPath(body)
+	require.False(t, isMatched, "Invalid xpath did not return false")
+}
+
+func TestMatcher_MatchXPath_XML(t *testing.T) {
+	body := `<?xml version="1.0" encoding="utf-8"?><foo>bar</foo><wibble id="1" /><parent><child>baz</child></parent>`
+	body2 := `<?xml version="1.0" encoding="utf-8"?><test>bar</test><wibble2 id="1" /><roditelj><dijete>alo</dijete></roditelj>`
+
+	// single match
+	m := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//foo[contains(text(), 'bar')]"}}
+	err := m.CompileMatchers()
+	require.Nil(t, err)
+
+	isMatched := m.MatchXPath(body)
+	require.True(t, isMatched, "Could not match valid XPath")
+
+	isMatched = m.MatchXPath("<h1>aaaaaaaaa</h1>")
+	require.False(t, isMatched, "Could match invalid XPath")
+
+	// OR match
+	m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "or", XPath: []string{"/foo[contains(text(), 'PATRICAAA')]", "/parent/child"}}
+	err = m.CompileMatchers()
+	require.Nil(t, err)
+
+	isMatched = m.MatchXPath(body)
+	require.True(t, isMatched, "Could not match valid multi-XPath with OR condition")
+
+	isMatched = m.MatchXPath(body2)
+	require.False(t, isMatched, "Could match invalid multi-XPath with OR condition")
+
+	// AND match
+	m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: "and", XPath: []string{"/foo[contains(text(), 'bar')]", "/parent/child"}}
+	err = m.CompileMatchers()
+	require.Nil(t, err)
+
+	isMatched = m.MatchXPath(body)
+	require.True(t, isMatched, "Could not match valid multi-XPath with AND condition")
+
+	isMatched = m.MatchXPath(body2)
+	require.False(t, isMatched, "Could match invalid multi-XPath with AND condition")
+
+	// invalid xpath
+	m = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{"//a[@a==1]"}}
+	_ = m.CompileMatchers()
+	isMatched = m.MatchXPath(body)
+	require.False(t, isMatched, "Invalid xpath did not return false")
+
+	// invalid xml
+	isMatched = m.MatchXPath("<h1> not right <q id=2/>notvalid")
+	require.False(t, isMatched, "Invalid xpath did not return false")
+}
diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go
index 2943c49455..670113d42c 100644
--- a/v2/pkg/operators/matchers/matchers.go
+++ b/v2/pkg/operators/matchers/matchers.go
@@ -94,6 +94,16 @@ type Matcher struct {
 	//       []string{"!contains(tolower(all_headers), ''strict-transport-security'')"}
 	DSL []string `yaml:"dsl,omitempty" json:"dsl,omitempty" jsonschema:"title=dsl expressions to match in response,description=DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"`
 	// description: |
+	//   XPath are the xpath queries expressions that will be evaluated against the response part.
+	// examples:
+	//   - name: XPath Matcher to check a title
+	//     value: >
+	//       []string{"/html/head/title[contains(text(), 'How to Find XPath')]"}
+	//   - name: XPath Matcher for finding links with target="_blank"
+	//     value: >
+	//       []string{"//a[@target="_blank"]"}
+	XPath []string `yaml:"xpath,omitempty" json:"xpath,omitempty" jsonschema:"title=xpath queries to match in response,description=xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"`
+	// description: |
 	//   Encoding specifies the encoding for the words field if any.
 	// values:
 	//   - "hex"
diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go
index 0ce6c7e692..9c872192a8 100644
--- a/v2/pkg/operators/matchers/matchers_types.go
+++ b/v2/pkg/operators/matchers/matchers_types.go
@@ -25,6 +25,8 @@ const (
 	SizeMatcher
 	// name:dsl
 	DSLMatcher
+	// name:xpath
+	XPathMatcher
 	limit
 )
 
@@ -36,6 +38,7 @@ var MatcherTypes = map[MatcherType]string{
 	RegexMatcher:  "regex",
 	BinaryMatcher: "binary",
 	DSLMatcher:    "dsl",
+	XPathMatcher:  "xpath",
 }
 
 // GetType returns the type of the matcher
diff --git a/v2/pkg/operators/matchers/validate.go b/v2/pkg/operators/matchers/validate.go
index 7d347919c1..9e0a7aba81 100644
--- a/v2/pkg/operators/matchers/validate.go
+++ b/v2/pkg/operators/matchers/validate.go
@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/antchfx/xpath"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	"gopkg.in/yaml.v3"
 )
@@ -38,8 +39,23 @@ func (matcher *Matcher) Validate() error {
 		expectedFields = append(commonExpectedFields, "Binary", "Part", "Encoding", "CaseInsensitive")
 	case RegexMatcher:
 		expectedFields = append(commonExpectedFields, "Regex", "Part", "Encoding", "CaseInsensitive")
+	case XPathMatcher:
+		expectedFields = append(commonExpectedFields, "XPath", "Part")
 	}
-	return checkFields(matcher, matcherMap, expectedFields...)
+
+	if err = checkFields(matcher, matcherMap, expectedFields...); err != nil {
+		return err
+	}
+
+	// validate the XPath query
+	if matcher.matcherType == XPathMatcher {
+		for _, query := range matcher.XPath {
+			if _, err = xpath.Compile(query); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }
 
 func checkFields(m *Matcher, matcherMap map[string]interface{}, expectedFields ...string) error {
diff --git a/v2/pkg/operators/matchers/validate_test.go b/v2/pkg/operators/matchers/validate_test.go
index e617a9491a..049ac045ff 100644
--- a/v2/pkg/operators/matchers/validate_test.go
+++ b/v2/pkg/operators/matchers/validate_test.go
@@ -15,4 +15,17 @@ func TestValidate(t *testing.T) {
 	m = &Matcher{matcherType: DSLMatcher, Part: "test"}
 	err = m.Validate()
 	require.NotNil(t, err, "Invalid template was correctly validated")
+
+	m = &Matcher{matcherType: XPathMatcher, XPath: []string{"//q[@id=\"foo\"]"}}
+
+	err = m.Validate()
+	require.Nil(t, err, "Could not validate correct XPath template")
+
+	m = &Matcher{matcherType: XPathMatcher, Status: []int{123}}
+	err = m.Validate()
+	require.NotNil(t, err, "Invalid XPath template was correctly validated")
+
+	m = &Matcher{matcherType: XPathMatcher, XPath: []string{"//a[@a==1]"}}
+	err = m.Validate()
+	require.NotNil(t, err, "Invalid XPath query was correctly validated")
 }
diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go
index d2b6be7fa9..d0cad29349 100644
--- a/v2/pkg/protocols/dns/operators.go
+++ b/v2/pkg/protocols/dns/operators.go
@@ -42,6 +42,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item)))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(types.ToString(item))), []string{}
 	}
 	return false, []string{}
 }
diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go
index 3b6c2af14f..2a7825dea2 100644
--- a/v2/pkg/protocols/file/operators.go
+++ b/v2/pkg/protocols/file/operators.go
@@ -30,6 +30,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
 	}
 	return false, []string{}
 }
diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go
index ddf5f672c6..ef552df3a5 100644
--- a/v2/pkg/protocols/headless/operators.go
+++ b/v2/pkg/protocols/headless/operators.go
@@ -37,6 +37,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
 	}
 	return false, []string{}
 }
diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go
index 41de04f846..03d75c7a62 100644
--- a/v2/pkg/protocols/http/operators.go
+++ b/v2/pkg/protocols/http/operators.go
@@ -40,6 +40,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(item)), []string{}
 	}
 	return false, []string{}
 }
diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go
index 38ffcfb1b3..5faf19797d 100644
--- a/v2/pkg/protocols/network/operators.go
+++ b/v2/pkg/protocols/network/operators.go
@@ -30,6 +30,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(itemStr)), []string{}
 	}
 	return false, []string{}
 }
diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go
index 007bfe4d0f..bccc8d19b2 100644
--- a/v2/pkg/protocols/offlinehttp/operators.go
+++ b/v2/pkg/protocols/offlinehttp/operators.go
@@ -40,6 +40,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), []string{}
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(item)), []string{}
 	}
 	return false, []string{}
 }
diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go
index 4b89af1bee..9e0d2a6ec4 100644
--- a/v2/pkg/protocols/protocols.go
+++ b/v2/pkg/protocols/protocols.go
@@ -205,6 +205,8 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher
 		return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
 	case matchers.DSLMatcher:
 		return matcher.Result(matcher.MatchDSL(data)), nil
+	case matchers.XPathMatcher:
+		return matcher.Result(matcher.MatchXPath(item)), []string{}
 	}
 	return false, nil
 }

From a2034cad51b68339434bb4b720f5a29485e01bb0 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Fri, 25 Aug 2023 17:14:14 +0000
Subject: [PATCH 4/6] Auto Generate Syntax Docs + JSONSchema [Fri Aug 25
 17:14:14 UTC 2023] :robot:

---
 nuclei-jsonschema.json | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 579db7082f..ff8291eb12 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -342,6 +342,14 @@
           "title": "dsl expressions to match in response",
           "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"
         },
+        "xpath": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "xpath queries to match in response",
+          "description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"
+        },
         "encoding": {
           "enum": [
             "hex"
@@ -371,7 +379,8 @@
         "binary",
         "status",
         "size",
-        "dsl"
+        "dsl",
+        "xpath"
       ],
       "type": "string",
       "title": "type of the matcher",

From 592a8a2fd54c3b1b9513d6569304223cbc455c89 Mon Sep 17 00:00:00 2001
From: Ice3man <nizamulrana@gmail.com>
Date: Sat, 26 Aug 2023 02:33:45 +0530
Subject: [PATCH 5/6] feat: added template-url support in template flag feature
 (#4089)

* misc docs update

* feat: added template-url support in template flag feature

* bugfix: added check URL condition

* template domain update

* editor host update

* misc update

* handle -turl template editor urls

* view remote templates using -td

* remove warning

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 README.md                              |  4 +-
 docs/editor/ai.mdx                     |  7 +--
 v2/cmd/integration-test/loader.go      | 10 ++--
 v2/cmd/nuclei/main.go                  |  6 +--
 v2/internal/runner/templates.go        |  6 +--
 v2/pkg/catalog/loader/loader.go        | 74 ++++++++++++++++++++++++++
 v2/pkg/catalog/loader/remote_loader.go |  1 -
 7 files changed, 87 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 7007930a03..8c770a234a 100644
--- a/README.md
+++ b/README.md
@@ -126,9 +126,9 @@ TEMPLATES:
    -ntv, -new-templates-version string[]  run new templates added in specific version
    -as, -automatic-scan                   automatic web scan using wappalyzer technology detection to tags mapping
    -t, -templates string[]                list of template or template directory to run (comma-separated, file)
-   -tu, -template-url string[]            list of template urls to run (comma-separated, file)
+   -turl, -template-url string[]          template url or list containing template urls to run (comma-separated, file)
    -w, -workflows string[]                list of workflow or workflow directory to run (comma-separated, file)
-   -wu, -workflow-url string[]            list of workflow urls to run (comma-separated, file)
+   -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
diff --git a/docs/editor/ai.mdx b/docs/editor/ai.mdx
index a5dee4172e..4824d138a6 100644
--- a/docs/editor/ai.mdx
+++ b/docs/editor/ai.mdx
@@ -6,14 +6,11 @@ title: 'AI Assistance'
 
 <img src="/images/ai.jpg" alt="AI Prompt" width="800px"/>
 
-[Nuclei Template Editor](https://templates.nuclei.sh/) employs AI to generate templates for vulnerability reports. This document seeks to guide you through the process, offering you usage tips and examples.
+[Nuclei Template Editor](https://templates.nuclei.sh/) has AI to generate templates for vulnerability reports. This document helps to guide you through the process, offering you usage tips and examples.
 
 ## Overview
 
-Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. It operates following these steps:
-
-1.  **Interpret a Prompt**: Analyzes a detailed prompt outlining a specific vulnerability.
-2.  **Generate a Template**: Creates a Nuclei template using the AI API.
+Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. First, the system interprets the user's prompt to identify a specific vulnerability. Then, it generates a template based on the steps required to reproduce the vulnerability along with all the necessary meta information to reproduce and remediate.
 
 ---
 
diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go
index d0ba4aac0b..92b682ce7b 100644
--- a/v2/cmd/integration-test/loader.go
+++ b/v2/cmd/integration-test/loader.go
@@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
 	if err == nil {
 		return fmt.Errorf("expected error for not allowed remote template list url")
 	}
@@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
@@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index 8edb9872bb..8c42eef256 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -137,14 +137,14 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", nil, "list of template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", nil, "list of workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
-		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"),
+		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"templates.nuclei.sh"}, "allowed domain list to load remote templates from"),
 	)
 
 	flagSet.CreateGroup("filters", "Filtering",
diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go
index 559448732f..f0d9cfe540 100644
--- a/v2/internal/runner/templates.go
+++ b/v2/internal/runner/templates.go
@@ -2,7 +2,6 @@ package runner
 
 import (
 	"bytes"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -45,14 +44,12 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 		if hasExtraFlags(r.options) {
 			if r.options.TemplateDisplay {
 				colorize := !r.options.NoColor
-
 				path := tpl.Path
-				tplBody, err := os.ReadFile(path)
+				tplBody, err := store.ReadTemplateFromURI(path, true)
 				if err != nil {
 					gologger.Error().Msgf("Could not read the template %s: %s", path, err)
 					continue
 				}
-
 				if colorize {
 					path = aurora.Cyan(tpl.Path).String()
 					tplBody, err = r.highlightTemplate(&tplBody)
@@ -60,7 +57,6 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 						gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err)
 						continue
 					}
-
 				}
 				gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody)
 			} else {
diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go
index 8535b82334..f633cbd7e0 100644
--- a/v2/pkg/catalog/loader/loader.go
+++ b/v2/pkg/catalog/loader/loader.go
@@ -1,8 +1,12 @@
 package loader
 
 import (
+	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
@@ -17,6 +21,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/retryablehttp-go"
+	errorutil "github.com/projectdiscovery/utils/errors"
+	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
+)
+
+const (
+	httpPrefix  = "http://"
+	httpsPrefix = "https://"
 )
 
 // Config contains the configuration options for the loader
@@ -120,6 +133,30 @@ func New(config *Config) (*Store, error) {
 		finalWorkflows: config.Workflows,
 	}
 
+	// Do a check to see if we have URLs in templates flag, if so
+	// we need to processs them separately and remove them from the initial list
+	var templatesFinal []string
+	for _, template := range config.Templates {
+		// TODO: Add and replace this with urlutil.IsURL() helper
+		if stringsutil.HasPrefixAny(template, httpPrefix, httpsPrefix) {
+			config.TemplateURLs = append(config.TemplateURLs, template)
+		} else {
+			templatesFinal = append(templatesFinal, template)
+		}
+	}
+
+	// fix editor paths
+	remoteTemplates := []string{}
+	for _, v := range config.TemplateURLs {
+		if _, err := urlutil.Parse(v); err == nil {
+			remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v))
+		} else {
+			templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
+		}
+	}
+	config.TemplateURLs = remoteTemplates
+	store.finalTemplates = templatesFinal
+
 	urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0
 	if urlBasedTemplatesProvided {
 		remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs, config.RemoteTemplateDomainList)
@@ -145,6 +182,43 @@ func New(config *Config) (*Store, error) {
 	return store, nil
 }
 
+func handleTemplatesEditorURLs(input string) string {
+	parsed, err := url.Parse(input)
+	if err != nil {
+		return input
+	}
+	if !strings.HasSuffix(parsed.Hostname(), "templates.nuclei.sh") {
+		return input
+	}
+	if strings.HasSuffix(parsed.Path, ".yaml") {
+		return input
+	}
+	parsed.Path = fmt.Sprintf("%s.yaml", parsed.Path)
+	finalURL := parsed.String()
+	return finalURL
+}
+
+// ReadTemplateFromURI should only be used for viewing templates
+// and should not be used anywhere else like loading and executing templates
+// there is no sandbox restriction here
+func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) {
+	if stringsutil.HasPrefixAny(uri, httpPrefix, httpsPrefix) && remote {
+		uri = handleTemplatesEditorURLs(uri)
+		remoteTemplates, _, err := getRemoteTemplatesAndWorkflows([]string{uri}, nil, store.config.RemoteTemplateDomainList)
+		if err != nil || len(remoteTemplates) == 0 {
+			return nil, errorutil.NewWithErr(err).Msgf("Could not load template %s: got %v", uri, remoteTemplates)
+		}
+		resp, err := retryablehttp.Get(remoteTemplates[0])
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		return io.ReadAll(resp.Body)
+	} else {
+		return os.ReadFile(uri)
+	}
+}
+
 // Templates returns all the templates in the store
 func (store *Store) Templates() []*templates.Template {
 	return store.templates
diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go
index c5e0a64324..77bfddaa2b 100644
--- a/v2/pkg/catalog/loader/remote_loader.go
+++ b/v2/pkg/catalog/loader/remote_loader.go
@@ -56,7 +56,6 @@ func getRemoteTemplatesAndWorkflows(templateURLs, workflowURLs, remoteTemplateDo
 			}
 		}
 	}
-
 	return remoteTemplateList, remoteWorkFlowList, err
 }
 

From 6d9d17cea26e3957ba7e913fb9f92abbd2cd469b Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 26 Aug 2023 13:55:29 +0530
Subject: [PATCH 6/6] version update + moved template changelog update with vv
 option

---
 v2/cmd/nuclei/main.go              | 4 ++--
 v2/internal/installer/template.go  | 2 +-
 v2/pkg/catalog/config/constants.go | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index 8c42eef256..91352b5ddc 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -341,9 +341,9 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
 
-	if options.Silent {
+	if options.VerboseVerbose {
 		// hide release notes if silent mode is enabled
-		installer.HideReleaseNotes = true
+		installer.HideReleaseNotes = false
 	}
 
 	if options.LeaveDefaultPorts {
diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go
index 1f514bbfbc..3db4924cc8 100644
--- a/v2/internal/installer/template.go
+++ b/v2/internal/installer/template.go
@@ -30,7 +30,7 @@ const (
 var (
 	HideProgressBar        = true
 	HideUpdateChangesTable = false
-	HideReleaseNotes       = false
+	HideReleaseNotes       = true
 )
 
 // TemplateUpdateResults contains the results of template update
diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go
index 830420d168..77a519ad2e 100644
--- a/v2/pkg/catalog/config/constants.go
+++ b/v2/pkg/catalog/config/constants.go
@@ -17,7 +17,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v2.9.12`
+	Version = `v2.9.13`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"