From 3dca03163ce2d5ff1f5d440ffadcd2e76d564b16 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Thu, 6 Jul 2023 18:03:52 +0200
Subject: [PATCH 01/12] Automatic target merge in network templates (#3904)

* skip visited actual addressess

* removed test

* adding disable clustering support
---
 integration_tests/network/same-address.yaml | 29 +++++++++++++++++++++
 v2/cmd/integration-test/network.go          |  1 +
 v2/pkg/protocols/network/request.go         |  9 ++++++-
 3 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 integration_tests/network/same-address.yaml

diff --git a/integration_tests/network/same-address.yaml b/integration_tests/network/same-address.yaml
new file mode 100644
index 0000000000..5cad9d21fb
--- /dev/null
+++ b/integration_tests/network/same-address.yaml
@@ -0,0 +1,29 @@
+id: same-target
+
+info:
+  name: same-target
+  author: pdteam
+  severity: info
+  description: Riak is a distributed NoSQL key-value data store that offers high availability, fault tolerance, operational simplicity, and scalability.
+
+network:
+  - host: 
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+      - "{{Hostname}}"
+    inputs:
+      - data: "PING\r\n"
+    read-size: 4
+    matchers:
+      - type: word
+        part: data
+        words:
+          - "PONG"
diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go
index 37ce8c3900..ff4d72a8c5 100644
--- a/v2/cmd/integration-test/network.go
+++ b/v2/cmd/integration-test/network.go
@@ -12,6 +12,7 @@ var networkTestcases = map[string]testutils.TestCase{
 	"network/multi-step.yaml":     &networkMultiStep{},
 	"network/self-contained.yaml": &networkRequestSelContained{},
 	"network/variables.yaml":      &networkVariables{},
+	"network/same-address.yaml":   &networkBasic{},
 }
 
 const defaultStaticPort = 5431
diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go
index 5acb518d7e..aecff19ea6 100644
--- a/v2/pkg/protocols/network/request.go
+++ b/v2/pkg/protocols/network/request.go
@@ -29,6 +29,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
 var _ protocols.Request = &Request{}
@@ -57,9 +58,16 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	variablesMap := request.options.Variables.Evaluate(variables)
 	variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
 
+	visitedAddressess := make(mapsutil.Map[string, struct{}])
+
 	for _, kv := range request.addresses {
 		actualAddress := replacer.Replace(kv.address, variables)
 
+		if visitedAddressess.Has(actualAddress) && !request.options.Options.DisableClustering {
+			continue
+		}
+		visitedAddressess.Set(actualAddress, struct{}{})
+
 		if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil {
 			outputEvent := request.responseToDSLMap("", "", "", address, "")
 			callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
@@ -110,7 +118,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		conn     net.Conn
 		err      error
 	)
-
 	if host, _, err := net.SplitHostPort(actualAddress); err == nil {
 		hostname = host
 	}

From 5e284879d3f54d6c29b2d9e1b7053c589f882347 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Fri, 7 Jul 2023 05:39:14 +0530
Subject: [PATCH 02/12] adds default ztls fallback support for tls errors
 (#3909)

* bump fastdialer

* make -ztls flag as no-op and deprecate

* bump fastdialer version from main branch

* dep update

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
 v2/cmd/nuclei/main.go                          | 2 +-
 v2/go.mod                                      | 2 +-
 v2/go.sum                                      | 4 ++--
 v2/pkg/protocols/common/protocolstate/state.go | 2 +-
 v2/pkg/protocols/ssl/ssl.go                    | 4 ----
 v2/pkg/types/types.go                          | 2 +-
 6 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index 3f93911e38..981db84995 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -201,7 +201,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"),
 		flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"),
 		flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"),
-		flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"),
+		flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all
 		flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"),
 		flagSet.BoolVar(&options.Sandbox, "sandbox", false, "sandbox nuclei for safe templates execution"),
 		flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
diff --git a/v2/go.mod b/v2/go.mod
index ffceed94c8..394f517955 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.19
-	github.com/projectdiscovery/fastdialer v0.0.32
+	github.com/projectdiscovery/fastdialer v0.0.33
 	github.com/projectdiscovery/hmap v0.0.13
 	github.com/projectdiscovery/interactsh v1.1.4
 	github.com/projectdiscovery/rawhttp v0.1.16
diff --git a/v2/go.sum b/v2/go.sum
index c63f90667c..b718e64189 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -397,8 +397,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z
 github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c=
 github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg=
 github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw=
-github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs=
-github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc=
+github.com/projectdiscovery/fastdialer v0.0.33 h1:FKXZjJme5nhgnnRL0Y4KjZ5YXKg03dX5ZJYFJV3LOwU=
+github.com/projectdiscovery/fastdialer v0.0.33/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go
index b4662aedab..4e9df494a3 100644
--- a/v2/pkg/protocols/common/protocolstate/state.go
+++ b/v2/pkg/protocols/common/protocolstate/state.go
@@ -95,8 +95,8 @@ func Init(options *types.Options) error {
 		opts.Deny = append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...)
 	}
 	opts.WithDialerHistory = true
-	opts.WithZTLS = options.ZTLS
 	opts.SNIName = options.SNI
+	// fastdialer now by default fallbacks to ztls when there are tls related errors
 	dialer, err := fastdialer.NewDialer(opts)
 	if err != nil {
 		return errors.Wrap(err, "could not create dialer")
diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go
index fe84fc1475..56972c01e2 100644
--- a/v2/pkg/protocols/ssl/ssl.go
+++ b/v2/pkg/protocols/ssl/ssl.go
@@ -112,10 +112,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	case request.ScanMode == "openssl" && !openssl.IsAvailable():
 		// if openssl is not installed instead of failing "auto" scanmode is used
 		request.ScanMode = "auto"
-
-	case options.Options.ZTLS && request.ScanMode == "ctls":
-		// only override if scanmode in template is "ctls" since auto internally uses ztls as fallback
-		request.ScanMode = "ztls"
 	}
 
 	tlsxOptions := &clients.Options{
diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
index 823ed858b8..b3880c05f1 100644
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@@ -277,7 +277,7 @@ type Options struct {
 	ClientKeyFile string
 	// ClientCAFile client certificate authority file (PEM-encoded) used for authenticating against scanned hosts
 	ClientCAFile string
-	// Use ZTLS library
+	// Deprecated: Use ZTLS library
 	ZTLS bool
 	// Sandbox enables sandboxed nuclei template execution
 	Sandbox bool

From 28f6c3797415290edc04b9fdec1ad37c5bc0559c Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Fri, 7 Jul 2023 23:35:32 +0530
Subject: [PATCH 03/12] misc fix

---
 v2/cmd/tmc/main.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go
index 97f8d6122b..ad75d08d60 100644
--- a/v2/cmd/tmc/main.go
+++ b/v2/cmd/tmc/main.go
@@ -206,11 +206,6 @@ func process(opts options) error {
 		}
 
 		if opts.enhance {
-			// currently enhance api only supports cve-id's
-			matches := idRegex.FindAllStringSubmatch(dataString, 1)
-			if len(matches) == 0 {
-				continue
-			}
 			enhancedTemplateData, isEnhanced, err := enhanceTemplate(dataString)
 			if err != nil {
 				gologger.Info().Label("enhance").Msg(logErrMsg(path, err, opts.debug, errFile))

From 9a44deb9e1d5375e0fe2f1c1faaf0e1b6f82972b Mon Sep 17 00:00:00 2001
From: Sandeep Singh <sandeep@projectdiscovery.io>
Date: Sun, 9 Jul 2023 01:19:56 +0530
Subject: [PATCH 04/12] added epss-percentile field under classification
 (#3911)

* added epss-percentile field under classification

* lint fix
---
 v2/cmd/tmc/main.go                         | 2 --
 v2/pkg/catalog/loader/filter/tag_filter.go | 1 +
 v2/pkg/model/model.go                      | 5 +++++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/v2/cmd/tmc/main.go b/v2/cmd/tmc/main.go
index ad75d08d60..c334372daf 100644
--- a/v2/cmd/tmc/main.go
+++ b/v2/cmd/tmc/main.go
@@ -76,8 +76,6 @@ func init() {
 	}
 }
 
-var idRegex = regexp.MustCompile("id: ([C|c][V|v][E|e]-[0-9]+-[0-9]+)")
-
 type options struct {
 	input        string
 	errorLogFile string
diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go
index 571c212af4..1da6090e35 100644
--- a/v2/pkg/catalog/loader/filter/tag_filter.go
+++ b/v2/pkg/catalog/loader/filter/tag_filter.go
@@ -207,6 +207,7 @@ func tryCollectConditionsMatchinfo(template *templates.Template) map[string]inte
 		parameters["cwe_id"] = template.Info.Classification.CWEID.ToSlice()
 		parameters["cpe"] = template.Info.Classification.CPE
 		parameters["epss_score"] = template.Info.Classification.EPSSScore
+		parameters["epss_percentile"] = template.Info.Classification.EPSSPercentile
 	}
 
 	if template.Type() == types.HTTPProtocol {
diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go
index 93e92f9664..63c0e1ee25 100644
--- a/v2/pkg/model/model.go
+++ b/v2/pkg/model/model.go
@@ -101,6 +101,11 @@ type Classification struct {
 	//   - value: "\"0.42509\""
 	EPSSScore float64 `json:"epss-score,omitempty" yaml:"epss-score,omitempty" jsonschema:"title=epss score for the template,description=EPSS Score for the template,example=0.42509"`
 	// description: |
+	//   EPSS Percentile for the template.
+	// examples:
+	//   - value: "\"0.42509\""
+	EPSSPercentile float64 `json:"epss-percentile,omitempty" yaml:"epss-percentile,omitempty" jsonschema:"title=epss percentile for the template,description=EPSS Percentile for the template,example=0.42509"`
+	// description: |
 	//   CPE for the template.
 	// examples:
 	//   - value: "\"cpe:/a:vendor:product:version\""

From 0993ac4cbbf7eed88444b90f487c0d0f12354f9b Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Sat, 8 Jul 2023 19:53:01 +0000
Subject: [PATCH 05/12] Auto Generate Syntax Docs + JSONSchema [Sat Jul  8
 19:53:01 UTC 2023] :robot:

---
 SYNTAX-REFERENCE.md               | 23 +++++++++++++++++++++++
 nuclei-jsonschema.json            |  5 +++++
 v2/pkg/templates/templates_doc.go | 19 +++++++++++++------
 3 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index def307bae2..a0406daa62 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -825,6 +825,29 @@ epss-score: "0.42509"
 ```
 
 
+</div>
+
+<hr />
+
+<div class="dd">
+
+<code>epss-percentile</code>  <i>float64</i>
+
+</div>
+<div class="dt">
+
+EPSS Percentile for the template.
+
+
+
+Examples:
+
+
+```yaml
+epss-percentile: "0.42509"
+```
+
+
 </div>
 
 <hr />
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 452d4fd853..bb6506ad89 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -32,6 +32,11 @@
           "title": "epss score for the template",
           "description": "EPSS Score for the template"
         },
+        "epss-percentile": {
+          "type": "number",
+          "title": "epss percentile for the template",
+          "description": "EPSS Percentile for the template"
+        },
         "cpe": {
           "type": "string",
           "title": "cpe for the template",
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go
index 25b094b01f..992e5bed6f 100644
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@@ -298,7 +298,7 @@ func init() {
 			FieldName: "classification",
 		},
 	}
-	MODELClassificationDoc.Fields = make([]encoder.Doc, 6)
+	MODELClassificationDoc.Fields = make([]encoder.Doc, 7)
 	MODELClassificationDoc.Fields[0].Name = "cve-id"
 	MODELClassificationDoc.Fields[0].Type = "stringslice.StringSlice"
 	MODELClassificationDoc.Fields[0].Note = ""
@@ -334,13 +334,20 @@ func init() {
 	MODELClassificationDoc.Fields[4].Comments[encoder.LineComment] = "EPSS Score for the template."
 
 	MODELClassificationDoc.Fields[4].AddExample("", "0.42509")
-	MODELClassificationDoc.Fields[5].Name = "cpe"
-	MODELClassificationDoc.Fields[5].Type = "string"
+	MODELClassificationDoc.Fields[5].Name = "epss-percentile"
+	MODELClassificationDoc.Fields[5].Type = "float64"
 	MODELClassificationDoc.Fields[5].Note = ""
-	MODELClassificationDoc.Fields[5].Description = "CPE for the template."
-	MODELClassificationDoc.Fields[5].Comments[encoder.LineComment] = "CPE for the template."
+	MODELClassificationDoc.Fields[5].Description = "EPSS Percentile for the template."
+	MODELClassificationDoc.Fields[5].Comments[encoder.LineComment] = "EPSS Percentile for the template."
 
-	MODELClassificationDoc.Fields[5].AddExample("", "cpe:/a:vendor:product:version")
+	MODELClassificationDoc.Fields[5].AddExample("", "0.42509")
+	MODELClassificationDoc.Fields[6].Name = "cpe"
+	MODELClassificationDoc.Fields[6].Type = "string"
+	MODELClassificationDoc.Fields[6].Note = ""
+	MODELClassificationDoc.Fields[6].Description = "CPE for the template."
+	MODELClassificationDoc.Fields[6].Comments[encoder.LineComment] = "CPE for the template."
+
+	MODELClassificationDoc.Fields[6].AddExample("", "cpe:/a:vendor:product:version")
 
 	HTTPRequestDoc.Type = "http.Request"
 	HTTPRequestDoc.Comments[encoder.LineComment] = " Request contains a http request to be made from a template"

From 2a9929878bdccd3c07b7b616b3375f6423df7abe Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 12 Jul 2023 05:15:01 +0530
Subject: [PATCH 06/12] chore(deps): bump golang.org/x/oauth2 from 0.9.0 to
 0.10.0 in /v2 (#3916)

Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.9.0 to 0.10.0.
- [Commits](https://github.com/golang/oauth2/compare/v0.9.0...v0.10.0)

---
updated-dependencies:
- dependency-name: golang.org/x/oauth2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 v2/go.mod | 12 ++++++------
 v2/go.sum | 26 +++++++++++++-------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/v2/go.mod b/v2/go.mod
index 394f517955..897d0f8aea 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -39,9 +39,9 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.11.0
-	golang.org/x/oauth2 v0.9.0
-	golang.org/x/text v0.10.0
+	golang.org/x/net v0.12.0
+	golang.org/x/oauth2 v0.10.0
+	golang.org/x/text v0.11.0
 	gopkg.in/yaml.v2 v2.4.0
 	moul.io/http2curl v1.0.0
 )
@@ -215,14 +215,14 @@ require (
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	goftp.io/server/v2 v2.0.0 // indirect
-	golang.org/x/crypto v0.10.0
+	golang.org/x/crypto v0.11.0
 	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.10.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.29.1 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
 )
diff --git a/v2/go.sum b/v2/go.sum
index b718e64189..a69243559f 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -617,8 +617,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -651,12 +651,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
-golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
-golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
+golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
+golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -704,8 +704,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -714,7 +714,7 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -725,8 +725,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -751,8 +751,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
-google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

From 5283d332142c05d337466328bdae9c3822597f41 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 12 Jul 2023 05:15:17 +0530
Subject: [PATCH 07/12] chore(deps): bump github.com/go-rod/rod from 0.113.0 to
 0.113.4 in /v2 (#3915)

Bumps [github.com/go-rod/rod](https://github.com/go-rod/rod) from 0.113.0 to 0.113.4.
- [Release notes](https://github.com/go-rod/rod/releases)
- [Commits](https://github.com/go-rod/rod/compare/v0.113.0...v0.113.4)

---
updated-dependencies:
- dependency-name: github.com/go-rod/rod
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 v2/go.mod | 2 +-
 v2/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/v2/go.mod b/v2/go.mod
index 897d0f8aea..dc5ea64c9f 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/bluele/gcache v0.0.2
 	github.com/corpix/uarand v0.2.0
 	github.com/go-playground/validator/v10 v10.14.1
-	github.com/go-rod/rod v0.113.0
+	github.com/go-rod/rod v0.113.4
 	github.com/gobwas/ws v1.2.1
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/itchyny/gojq v0.12.13
diff --git a/v2/go.sum b/v2/go.sum
index a69243559f..aeb9cb5c99 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -187,8 +187,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=
 github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-rod/rod v0.113.0 h1:E7+GLjYVZnScewIB2u8+66joQLaDGbOLzSOT4orNHms=
-github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
+github.com/go-rod/rod v0.113.4 h1:O5a/VTl1h6a08ecXkOF9oVJBi8fjBnF0FtyMSgd3KaY=
+github.com/go-rod/rod v0.113.4/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
 github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
 github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=

From 7eadcf761561d9c7a6f83b7b5b361b195461fe1b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 12 Jul 2023 05:15:32 +0530
Subject: [PATCH 08/12] chore(deps): bump
 github.com/projectdiscovery/wappalyzergo in /v2 (#3914)

Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.102 to 0.0.104.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.102...v0.0.104)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 v2/go.mod | 2 +-
 v2/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/v2/go.mod b/v2/go.mod
index dc5ea64c9f..6dcacccd02 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -78,7 +78,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.0
 	github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1
 	github.com/projectdiscovery/utils v0.0.40
-	github.com/projectdiscovery/wappalyzergo v0.0.102
+	github.com/projectdiscovery/wappalyzergo v0.0.104
 	github.com/stretchr/testify v1.8.4
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v3 v3.0.1
diff --git a/v2/go.sum b/v2/go.sum
index aeb9cb5c99..d2dacbe479 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -436,8 +436,8 @@ github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6L
 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8=
 github.com/projectdiscovery/utils v0.0.40 h1:vChjk6PnSb6+nUaP3/GS8LjbGGlrUpFiZBbeoBa9Sww=
 github.com/projectdiscovery/utils v0.0.40/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo=
-github.com/projectdiscovery/wappalyzergo v0.0.102 h1:ABjZghof2U2yzGNL+q5ouWHEardLd2o53Ukgrf8CZzE=
-github.com/projectdiscovery/wappalyzergo v0.0.102/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
+github.com/projectdiscovery/wappalyzergo v0.0.104 h1:hdda6WxAzXVpLBbJW1sLqrwOXHn0prP9IYFY7dfCMjE=
+github.com/projectdiscovery/wappalyzergo v0.0.104/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
 github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
 github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8=

From 880adc4405fa4b5cc5c259d67254b3b2f39a5bfb Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Wed, 12 Jul 2023 02:54:55 +0300
Subject: [PATCH 09/12] Automate merging of dependabot PRs to dep branch
 (#3857)

* add auto sync and merge workflows

* misc update

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
 .github/dependabot.yml           |  6 +++---
 .github/workflows/auto-merge.yml | 25 +++++++++++++++++++++++++
 .github/workflows/sync-dep.yml   | 30 ++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/auto-merge.yml
 create mode 100644 .github/workflows/sync-dep.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index fddde924e8..24bdcc299b 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -11,7 +11,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    target-branch: "dev"
+    target-branch: "dep"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -23,7 +23,7 @@ updates:
     directory: "v2/"
     schedule:
       interval: "weekly"
-    target-branch: "dev"
+    target-branch: "dep"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -35,7 +35,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    target-branch: "dev"
+    target-branch: "dep"
     commit-message:
       prefix: "chore"
       include: "scope"
diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml
new file mode 100644
index 0000000000..0e56f0a03c
--- /dev/null
+++ b/.github/workflows/auto-merge.yml
@@ -0,0 +1,25 @@
+name: 🤖 dep auto merge
+
+on:
+  pull_request:
+    branches:
+      - dep
+  workflow_dispatch:
+
+permissions:
+  pull-requests: write
+  issues: write
+  repository-projects: write
+
+jobs:
+  automerge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.DEPENDABOT_PAT }}
+
+      - uses: ahmadnassri/action-dependabot-auto-merge@v2
+        with:
+          github-token: ${{ secrets.DEPENDABOT_PAT }}
+          target: all
\ No newline at end of file
diff --git a/.github/workflows/sync-dep.yml b/.github/workflows/sync-dep.yml
new file mode 100644
index 0000000000..dfe9af9af9
--- /dev/null
+++ b/.github/workflows/sync-dep.yml
@@ -0,0 +1,30 @@
+name: 🤖 sync dep
+
+on:
+  push:
+    branches:
+      - dev
+
+permissions:
+  pull-requests: write
+  issues: write
+  repository-projects: write
+
+jobs:
+  sync-dep-with-dev:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 12
+
+      - name: Opening pull request
+        id: pull
+        uses: tretuna/sync-branches@1.4.0
+        with:
+          GITHUB_TOKEN: ${{ secrets.DEPENDABOT_PAT }}
+          FROM_BRANCH: "${{ github.ref_name }}"
+          TO_BRANCH: "dep"
\ No newline at end of file

From eaa6209a77af89ebd4e31f6af00196bed617173d Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Wed, 12 Jul 2023 05:27:57 +0530
Subject: [PATCH 10/12] misc update

---
 .github/workflows/sync-dep.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sync-dep.yml b/.github/workflows/sync-dep.yml
index dfe9af9af9..73798b78dc 100644
--- a/.github/workflows/sync-dep.yml
+++ b/.github/workflows/sync-dep.yml
@@ -25,6 +25,6 @@ jobs:
         id: pull
         uses: tretuna/sync-branches@1.4.0
         with:
-          GITHUB_TOKEN: ${{ secrets.DEPENDABOT_PAT }}
+          GITHUB_TOKEN: "${{ secrets.DEPENDABOT_PAT }}"
           FROM_BRANCH: "${{ github.ref_name }}"
           TO_BRANCH: "dep"
\ No newline at end of file

From 4d1c9fe8b1d560f7614919bab03fb625cfc99761 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 12 Jul 2023 05:30:20 +0530
Subject: [PATCH 11/12] chore(deps): bump github.com/projectdiscovery/ratelimit
 in /v2 (#3913)

Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.8 to 0.0.9.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.8...v0.0.9)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 v2/go.mod | 2 +-
 v2/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/v2/go.mod b/v2/go.mod
index 6dcacccd02..0c669bdc58 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -72,7 +72,7 @@ require (
 	github.com/projectdiscovery/gologger v1.1.10
 	github.com/projectdiscovery/httpx v1.3.0
 	github.com/projectdiscovery/mapcidr v1.1.2
-	github.com/projectdiscovery/ratelimit v0.0.8
+	github.com/projectdiscovery/ratelimit v0.0.9
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.0
diff --git a/v2/go.sum b/v2/go.sum
index d2dacbe479..8593fb82e2 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -417,8 +417,8 @@ github.com/projectdiscovery/mapcidr v1.1.2 h1:Mmq/nPqvVc7fjvH/kJVK0IBOny/LrJIxZ4
 github.com/projectdiscovery/mapcidr v1.1.2/go.mod h1:Aoq0x/wJl6KDbtQ8OcPkjIDCqx2iEyx5ty1nzso8wXM=
 github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o=
 github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk=
-github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI=
-github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic=
+github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8=
+github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I=
 github.com/projectdiscovery/rawhttp v0.1.16 h1:2wggRuVUty2Sypy1MKWM/NNmfbC97WkG6YtmoxzCOVc=
 github.com/projectdiscovery/rawhttp v0.1.16/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=

From d51e058791db22ac3f471544a5fbab4f3be15ccb Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Thu, 13 Jul 2023 00:51:06 +0530
Subject: [PATCH 12/12] add ErrNoMoreRequests for generator (#3918)

* add ErrNoMoreRequests for generator

* fix gh repo name convention

* fix dirname in unit test
---
 v2/pkg/external/customtemplates/github.go      | 16 ++--------------
 v2/pkg/external/customtemplates/github_test.go |  2 +-
 v2/pkg/protocols/common/fuzz/parts.go          |  6 ++----
 v2/pkg/protocols/common/generators/load.go     |  4 ++--
 v2/pkg/protocols/headless/request.go           |  4 ++--
 v2/pkg/protocols/http/build_request.go         |  2 +-
 v2/pkg/protocols/http/request.go               |  6 +++---
 v2/pkg/protocols/http/utils/requtils.go        |  1 +
 v2/pkg/types/types.go                          |  6 ++++++
 9 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go
index e0cd75d9ce..d9317ca3af 100644
--- a/v2/pkg/external/customtemplates/github.go
+++ b/v2/pkg/external/customtemplates/github.go
@@ -150,21 +150,9 @@ func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) e
 	return nil
 }
 
-// getLocalRepoClonePath returns the clone path.
-// if same name repo directory exists from another owner then it appends the owner then and returns the path
-// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep
+// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness
 func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string {
-	if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) {
-		return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
-	}
-	return filepath.Join(downloadPath, ctr.reponame)
-}
-
-// isRepoDirExists take the path and checks if the same repo or not
-func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool {
-	r, _ := git.PlainOpen(repoPath)
-	local, _ := r.Config()
-	return local.User.Name == ctr.owner // repo already cloned no need to rename and clone
+	return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
 }
 
 // returns the auth object with username and github token as password
diff --git a/v2/pkg/external/customtemplates/github_test.go b/v2/pkg/external/customtemplates/github_test.go
index 60d469565a..e21a62a90f 100644
--- a/v2/pkg/external/customtemplates/github_test.go
+++ b/v2/pkg/external/customtemplates/github_test.go
@@ -30,6 +30,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) {
 
 	ctm.Download(context.Background())
 
-	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists")
+	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists")
 	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists")
 }
diff --git a/v2/pkg/protocols/common/fuzz/parts.go b/v2/pkg/protocols/common/fuzz/parts.go
index 43e0f4201b..0e09aaf802 100644
--- a/v2/pkg/protocols/common/fuzz/parts.go
+++ b/v2/pkg/protocols/common/fuzz/parts.go
@@ -2,14 +2,13 @@ package fuzz
 
 import (
 	"context"
-	"io"
 	"net/http"
 	"strings"
 
 	"github.com/corpix/uarand"
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -49,7 +48,6 @@ func (rule *Rule) executeQueryPartRule(input *ExecuteRuleInput, payload string)
 				requestURL.Params = temp
 				if qerr := rule.buildQueryInput(input, requestURL, input.InteractURLs); qerr != nil {
 					err = qerr
-					gologger.Error().Msgf("Could not build request for query part rule %v: %s\n", rule, err)
 					return false
 				}
 				cloned[i] = value // change back to previous value for temp
@@ -89,7 +87,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
 		DynamicValues: input.Values,
 	}
 	if !input.Callback(request) {
-		return io.EOF
+		return types.ErrNoMoreRequests
 	}
 	return nil
 }
diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go
index de419488e3..612f4e33e3 100644
--- a/v2/pkg/protocols/common/generators/load.go
+++ b/v2/pkg/protocols/common/generators/load.go
@@ -2,11 +2,11 @@ package generators
 
 import (
 	"bufio"
-	"io"
 	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
+	pkgTypes "github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/spf13/cast"
 )
 
@@ -60,7 +60,7 @@ func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]stri
 		}
 		lines = append(lines, text)
 	}
-	if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) {
+	if err := scanner.Err(); err != nil && !errors.Is(err, pkgTypes.ErrNoMoreRequests) {
 		return lines, scanner.Err()
 	}
 	return lines, nil
diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go
index 9365d142cf..b7f40eb415 100644
--- a/v2/pkg/protocols/headless/request.go
+++ b/v2/pkg/protocols/headless/request.go
@@ -1,7 +1,6 @@
 package headless
 
 import (
-	"io"
 	"net/url"
 	"strings"
 	"time"
@@ -22,6 +21,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -206,7 +206,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
 			Values:      payloads,
 			BaseRequest: nil,
 		})
-		if err == io.EOF {
+		if err == types.ErrNoMoreRequests {
 			return nil
 		}
 		if err != nil {
diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go
index 1b76840261..eecd9eadf4 100644
--- a/v2/pkg/protocols/http/build_request.go
+++ b/v2/pkg/protocols/http/build_request.go
@@ -65,7 +65,7 @@ func (r *requestGenerator) Total() int {
 }
 
 // Make creates a http request for the provided input.
-// It returns io.EOF as error when all the requests have been exhausted.
+// It returns ErrNoMoreRequests as error when all the requests have been exhausted.
 func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	// value of `reqData` depends on the type of request specified in template
 	// 1. If request is raw request =  reqData contains raw request (i.e http request dump)
diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go
index 127b7a0ee4..04469edc1f 100644
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@@ -134,7 +134,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
-			if err == io.EOF {
+			if err == types.ErrNoMoreRequests {
 				break
 			}
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
@@ -301,7 +301,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 				Values:      generated.dynamicValues,
 				BaseRequest: generated.request,
 			})
-			if err == io.EOF {
+			if err == types.ErrNoMoreRequests {
 				return nil
 			}
 			if err != nil {
@@ -354,7 +354,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			defer cancel()
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
 			if err != nil {
-				if err == io.EOF {
+				if err == types.ErrNoMoreRequests {
 					return true, nil
 				}
 				request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
diff --git a/v2/pkg/protocols/http/utils/requtils.go b/v2/pkg/protocols/http/utils/requtils.go
index cfb75c1547..7be1f22591 100644
--- a/v2/pkg/protocols/http/utils/requtils.go
+++ b/v2/pkg/protocols/http/utils/requtils.go
@@ -13,6 +13,7 @@ var (
 	urlWithPortRegex = regexp.MustCompile(`^{{(BaseURL|RootURL)}}:(\d+)`)
 	// regex to detect traling slash in path (not applicable to raw requests)
 	trailingSlashregex = regexp.MustCompile(`^\Q{{\E[a-zA-Z]+\Q}}/\E`)
+	// ErrNoMoreRequests is internal error to
 )
 
 // HasTrailingSlash returns true if path(that has default variables) has trailing slash
diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
index b3880c05f1..ada0aed728 100644
--- a/v2/pkg/types/types.go
+++ b/v2/pkg/types/types.go
@@ -1,6 +1,7 @@
 package types
 
 import (
+	"io"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -9,6 +10,11 @@ import (
 	fileutil "github.com/projectdiscovery/utils/file"
 )
 
+var (
+	// ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate
+	ErrNoMoreRequests = io.EOF
+)
+
 // Options contains the configuration options for nuclei scanner.
 type Options struct {
 	// Tags contains a list of tags to execute templates for. Multiple paths