From 38f448d352b92a2a8267e9afdc3ff46bc05d5f4a Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Sun, 2 Apr 2023 15:26:32 +0530 Subject: [PATCH 01/34] Update config.go --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index b7cdf6995e..ad0b04cc7e 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -32,7 +32,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.9.1` +const Version = `2.9.2-dev` var customConfigDirectory string From a1853481946a0c9af36f619b95b5e83808f98f1b Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Tue, 4 Apr 2023 07:39:52 +0530 Subject: [PATCH 02/34] fix integration test (#3506) * fix integration test * fix interactsh fatal error --- v2/cmd/integration-test/fuzz.go | 4 ++-- v2/cmd/integration-test/integration-test.go | 4 +++- v2/go.mod | 8 ++++---- v2/go.sum | 16 ++++++++-------- v2/pkg/testutils/integration.go | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/v2/cmd/integration-test/fuzz.go b/v2/cmd/integration-test/fuzz.go index 38fa04a939..8fe4b95810 100644 --- a/v2/cmd/integration-test/fuzz.go +++ b/v2/cmd/integration-test/fuzz.go @@ -50,7 +50,7 @@ func (h *fuzzModeOverride) Execute(filePath string) error { }) ts := httptest.NewTLSServer(router) defer ts.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", false, "-fuzzing-mode", "single", "-json") + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", debug, "-fuzzing-mode", "single", "-jsonl") if err != nil { return err } @@ -95,7 +95,7 @@ func (h *fuzzTypeOverride) Execute(filePath string) error { }) ts := httptest.NewTLSServer(router) defer ts.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", false, "-fuzzing-type", "replace", "-json") + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", debug, "-fuzzing-type", "replace", "-jsonl") if err != nil { return err } diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index c08deb5125..3d0411decf 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -128,5 +128,7 @@ func expectResultsCount(results []string, expectedNumber int) error { } func normalizeSplit(str string) []string { - return strings.Split(strings.TrimSpace(str), ",") + return strings.FieldsFunc(str, func(r rune) bool { + return r == ',' + }) } diff --git a/v2/go.mod b/v2/go.mod index 74cd7e49a6..5862c64c10 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -20,16 +20,16 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.52 + github.com/miekg/dns v1.1.53 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.12 github.com/projectdiscovery/fastdialer v0.0.24 github.com/projectdiscovery/hmap v0.0.11 - github.com/projectdiscovery/interactsh v1.1.2 + github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d github.com/projectdiscovery/rawhttp v0.1.10 github.com/projectdiscovery/retryabledns v1.0.21 - github.com/projectdiscovery/retryablehttp-go v1.0.13 + github.com/projectdiscovery/retryablehttp-go v1.0.14 github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 @@ -43,7 +43,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.30.0 github.com/xanzy/go-gitlab v0.80.2 - go.uber.org/multierr v1.10.0 + go.uber.org/multierr v1.11.0 golang.org/x/net v0.8.0 golang.org/x/oauth2 v0.6.0 golang.org/x/text v0.8.0 diff --git a/v2/go.sum b/v2/go.sum index 4bcc47dd4e..14802253fd 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -372,8 +372,8 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.52 h1:Bmlc/qsNNULOe6bpXcUTsuOajd0DzRHwup6D9k1An0c= -github.com/miekg/dns v1.1.52/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= +github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= @@ -442,8 +442,8 @@ github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2n github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI= github.com/projectdiscovery/httpx v1.2.9 h1:bSXXjPHIDywokASSXLaWScyIRTFT6Avr9JJS7lV96c0= github.com/projectdiscovery/httpx v1.2.9/go.mod h1:99+TTGdMDKQgWRLzzWj3zGffQ6f4NCAWQHqHWfPn0Uc= -github.com/projectdiscovery/interactsh v1.1.2 h1:CtnVd8vgn6wRK1A2dD77XuGUOmsz5e/bEozzRme9oJs= -github.com/projectdiscovery/interactsh v1.1.2/go.mod h1:aoNDXOSGlMTP0ce9sgiE2tEX3+P+iE9CPtzc2cYONl8= +github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d h1:V/TpFiLbpSsgZgxBXSTXSgwGD8uKayImAwqz9etAzK8= +github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d/go.mod h1:6qYsmXQ//CPXib21ugKYXuLwr43zLPlO+UYyx+o+4Mw= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e1WPXpluc0= @@ -460,8 +460,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.21 h1:vOpPQR1q8Z824uoA8JXCI/RyvDAssPeD68Onz9hP/ds= github.com/projectdiscovery/retryabledns v1.0.21/go.mod h1:6oTPKMRlKZ7lIIEzTH723K6RvNRjmm6fe9br4Dom3UI= -github.com/projectdiscovery/retryablehttp-go v1.0.13 h1:gKxd/J08Dxc8a/LFvTz9+JUedEvivH3PoDnQQEHAY4M= -github.com/projectdiscovery/retryablehttp-go v1.0.13/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= +github.com/projectdiscovery/retryablehttp-go v1.0.14 h1:PJaHQtHWE00xrmryZwhtma2b72GbkfA9gJM0yIR9ENY= +github.com/projectdiscovery/retryablehttp-go v1.0.14/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -633,8 +633,8 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index bf973dfa46..b3a90c5d0c 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -64,7 +64,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro fmt.Println(string(data)) } if err != nil { - return nil, err + return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) } var parts []string items := strings.Split(string(data), "\n") From cd182aa1e2b75bdd022c564fc13581de45ab7961 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 01:47:17 +0800 Subject: [PATCH 03/34] chore(deps): bump alpine from 3.17.2 to 3.17.3 (#3497) Bumps alpine from 3.17.2 to 3.17.3. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 65d4150f5f..255bf68f18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN go mod download RUN go build ./cmd/nuclei # Release -FROM alpine:3.17.2 +FROM alpine:3.17.3 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools chromium ca-certificates COPY --from=build-env /app/v2/nuclei /usr/local/bin/ From ca58df668a63c6fc4b7cc4415a1a110973f453fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 01:48:19 +0800 Subject: [PATCH 04/34] chore(deps): bump github.com/projectdiscovery/rawhttp in /v2 (#3498) Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.10 to 0.1.11. - [Release notes](https://github.com/projectdiscovery/rawhttp/releases) - [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.10...v0.1.11) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/rawhttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5862c64c10..a789dfe91a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -27,7 +27,7 @@ require ( github.com/projectdiscovery/fastdialer v0.0.24 github.com/projectdiscovery/hmap v0.0.11 github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d - github.com/projectdiscovery/rawhttp v0.1.10 + github.com/projectdiscovery/rawhttp v0.1.11 github.com/projectdiscovery/retryabledns v1.0.21 github.com/projectdiscovery/retryablehttp-go v1.0.14 github.com/projectdiscovery/stringsutil v0.0.2 // indirect diff --git a/v2/go.sum b/v2/go.sum index 14802253fd..5126d679f9 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -454,8 +454,8 @@ github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/rawhttp v0.1.10 h1:wkQk/lpMVzi4AAELRDaBQEgMqyerpkz3Kks7QgDF274= -github.com/projectdiscovery/rawhttp v0.1.10/go.mod h1:cIlAWs3Nu8CTBArx/8GU1baimR5T1eO62TJFG2rAnSc= +github.com/projectdiscovery/rawhttp v0.1.11 h1:NbfunXIVdmFWAhZ864fx09sQLnHOVTYKhAe9P2Cnass= +github.com/projectdiscovery/rawhttp v0.1.11/go.mod h1:aH95fdyjmvUmyQWPTnIcHv5ncjhisOki76nXqXZZYYg= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.21 h1:vOpPQR1q8Z824uoA8JXCI/RyvDAssPeD68Onz9hP/ds= From 42168765b34018c26907579f1bf83ad91e633579 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 01:50:00 +0800 Subject: [PATCH 05/34] chore(deps): bump github.com/xanzy/go-gitlab in /v2 (#3499) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.80.2 to 0.81.0. - [Release notes](https://github.com/xanzy/go-gitlab/releases) - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.80.2...v0.81.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 4 ++-- v2/go.sum | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index a789dfe91a..71582dd6aa 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -42,7 +42,7 @@ require ( github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.30.0 - github.com/xanzy/go-gitlab v0.80.2 + github.com/xanzy/go-gitlab v0.81.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.8.0 golang.org/x/oauth2 v0.6.0 @@ -170,7 +170,7 @@ require ( github.com/gosuri/uilive v0.0.4 // indirect github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hdm/jarm-go v0.0.7 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect diff --git a/v2/go.sum b/v2/go.sum index 5126d679f9..e80354d5ee 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -252,13 +252,12 @@ github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJS github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= -github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -590,8 +589,8 @@ github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= -github.com/xanzy/go-gitlab v0.80.2 h1:CH1Q7NDklqZllox4ICVF4PwlhQGfPtE+w08Jsb74ZX0= -github.com/xanzy/go-gitlab v0.80.2/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M= +github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk= +github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= From a41b61e76105a60ccf293571958a978c384a1e9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 01:50:35 +0800 Subject: [PATCH 06/34] chore(deps): bump github.com/aws/aws-sdk-go-v2 in /v2 (#3500) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.17.6 to 1.17.7. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.17.6...v1.17.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 71582dd6aa..4a7d91ed85 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -55,7 +55,7 @@ require ( github.com/DataDog/gostackparse v0.6.0 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 - github.com/aws/aws-sdk-go-v2 v1.17.6 + github.com/aws/aws-sdk-go-v2 v1.17.7 github.com/aws/aws-sdk-go-v2/config v1.18.15 github.com/aws/aws-sdk-go-v2/credentials v1.13.17 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55 diff --git a/v2/go.sum b/v2/go.sum index e80354d5ee..01915b82c7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -62,8 +62,9 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.17.6 h1:Y773UK7OBqhzi5VDXMi1zVGsoj+CVHs2eaC2bDsLwi0= github.com/aws/aws-sdk-go-v2 v1.17.6/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY= From fd6677d16bbb5a3212e15a57abb296562dd39ebd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 01:50:57 +0800 Subject: [PATCH 07/34] chore(deps): bump github.com/go-git/go-git/v5 from 5.5.2 to 5.6.1 in /v2 (#3501) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.5.2 to 5.6.1. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.5.2...v5.6.1) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 10 +++++----- v2/go.sum | 34 +++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 4a7d91ed85..e957830a33 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -62,7 +62,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 github.com/docker/go-units v0.5.0 github.com/fatih/structs v1.1.0 - github.com/go-git/go-git/v5 v5.5.2 + github.com/go-git/go-git/v5 v5.6.1 github.com/h2non/filetype v1.1.3 github.com/klauspost/compress v1.16.0 github.com/labstack/echo/v4 v4.10.2 @@ -118,7 +118,7 @@ require ( github.com/minio/selfupdate v0.6.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect - github.com/pjbgf/sha1cd v0.2.3 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/projectdiscovery/asnmap v1.0.2 // indirect github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect github.com/projectdiscovery/freeport v0.0.4 // indirect @@ -222,8 +222,8 @@ require ( require ( github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect github.com/alecthomas/chroma v0.10.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect @@ -236,7 +236,7 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-git/go-billy/v5 v5.4.0 // indirect + github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.13 // indirect diff --git a/v2/go.sum b/v2/go.sum index 01915b82c7..308c715d67 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -14,16 +14,16 @@ github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb888350 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo= github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:AqtPw7WNT0O69k+AbPKWVGYeW94TqgMW/g+Ppc8AZr4= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= -github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= +github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= @@ -179,12 +179,12 @@ github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4x github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE= -github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= +github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= +github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw= -github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI= +github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= +github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -380,6 +380,7 @@ github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeB github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -409,8 +410,8 @@ github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtb github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI= -github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -640,6 +641,7 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= goftp.io/server/v2 v2.0.0 h1:FF8JKXXKDxAeO1uXEZz7G+IZwCDhl19dpVIlDtp3QAg= goftp.io/server/v2 v2.0.0/go.mod h1:7+H/EIq7tXdfo1Muu5p+l3oQ6rYkDZ8lY7IM5d5kVdQ= +golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -656,7 +658,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= @@ -665,6 +668,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -687,8 +691,9 @@ golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -738,6 +743,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -749,7 +755,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= @@ -773,6 +779,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -825,3 +832,4 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From 5602630a2eca6028f2ba81c1e4a57e0e34f39c0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 02:52:09 +0800 Subject: [PATCH 08/34] chore(deps): bump github.com/aws/aws-sdk-go-v2/config in /v2 (#3502) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.15 to 1.18.19. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.15...config/v1.18.19) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 20 ++++++++++---------- v2/go.sum | 39 ++++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e957830a33..087fa26b63 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -56,8 +56,8 @@ require ( github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go-v2 v1.17.7 - github.com/aws/aws-sdk-go-v2/config v1.18.15 - github.com/aws/aws-sdk-go-v2/credentials v1.13.17 + github.com/aws/aws-sdk-go-v2/config v1.18.19 + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55 github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 github.com/docker/go-units v0.5.0 @@ -225,14 +225,14 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/alecthomas/chroma v0.10.0 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 308c715d67..87f8f0a12f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -62,29 +62,30 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.17.6/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY= github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= -github.com/aws/aws-sdk-go-v2/credentials v1.13.17 h1:IubQO/RNeIVKF5Jy77w/LfUvmmCxTnk2TP1UZZIMiF4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.17/go.mod h1:K9xeFo1g/YPMguMUD69YpwB4Nyi6W/5wn706xIInJFg= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 h1:/2Cb3SK3xVOQA7Xfr5nCWCo5H3UiNINtsVvVdk8sQqA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0/go.mod h1:neYVaeKr5eT7BzwULuG2YbLhzWZ22lpjKdCybR7AXrQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55 h1:ClZKHmu2QIRQCEQ2Y2upfu4JPO0pG69Ce5eiq3PS2V4= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55/go.mod h1:L/h5B6I7reig2QJXCGY0e0NVx4hYCcjETmsfR02hFng= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 h1:y+8n9AGDjikyXoMBTRaHHHSaFEB8267ykmvyPodJfys= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30/go.mod h1:LUBAO3zNXQjoONBKn/kR1y0Q4cj/D02Ts0uHYjcCQLM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 h1:r+Kv+SEJquhAZXaJ7G4u44cIwXV3f8K+N482NNAzJZA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24/go.mod h1:gAuCezX/gob6BSMbItsSlMb6WZGV7K2+fWOvk8xBSto= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 h1:IVx9L7YFhpPq0tTnGo8u8TpluFu7nAn9X3sUDMb11c0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21 h1:QdxdY43AiwsqG/VAqHA7bIVSm3rKr8/p9i05ydA0/RM= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21/go.mod h1:QtIEat7ksHH8nFItljyvMI0dGj8lipK2XZ4PhNihTEU= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= @@ -92,21 +93,21 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1: github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24 h1:Qmm8klpAdkuN3/rPrIMa/hZQ1z93WMBPjOzdAsbSnlo= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24/go.mod h1:QelGeWBVRh9PbbXsfXKTFlU9FjT6W2yP+dW5jMQzOkg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 h1:c5qGfdbCHav6viBwiyDns3OXqhqAbGjfIB4uVu2ayhk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24/go.mod h1:HMA4FZG6fyib+NDo5bpIxX1EhYjrAOveZJY2YR0xrNE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23 h1:qc+RW0WWZ2KApMnsu/EVCPqLTyIH55uc7YQq7mq4XqE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23/go.mod h1:FJhZWVWBCcgAF8jbep7pxQ1QUsjzTwa9tvEXGw2TDRo= github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 h1:kFfb+NMap4R7nDvBYyABa/nw7KFMtAfygD1Hyoxh4uE= github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5/go.mod h1:Dze3kNt4T+Dgb8YCfuIFSBLmE6hadKNxqfdF0Xmqz1I= github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 h1:bdKIX6SVF3nc3xJFw6Nf0igzS6Ff/louGq8Z6VP/3Hs= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.5/go.mod h1:vuWiaDB30M/QTC+lI3Wj6S/zb7tpUK2MSYgy3Guh2L0= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 h1:xLPZMyuZ4GuqRCIec/zWuIhRFPXh2UOJdLXBSi64ZWQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5/go.mod h1:QjxpHmCwAg0ESGtPQnLIVp7SedTOBMYy+Slr3IfMKeI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 h1:rIFn5J3yDoeuKCE9sESXqM5POTAhOP1du3bv/qTL+tE= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.6/go.mod h1:48WJ9l3dwP0GSHWGc5sFGGlCkuA82Mc2xnw+T6Q8aDw= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= From 3476f4d1d682c2733470fab5e537812cb416872d Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Sat, 8 Apr 2023 08:14:41 -0400 Subject: [PATCH 09/34] JSONL(ine) Export (#3504) (#3505) * Add initial hooks for JSONL export * Add newline character after each result * fix integration test (#3506) * fix integration test * fix interactsh fatal error * fix default report-config.yaml --------- Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Co-authored-by: Tarun Koyalwar --- README.md | 3 +- README_ID.md | 3 +- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 12 +++- v2/pkg/reporting/exporters/jsonl/jsonl.go | 80 +++++++++++++++++++++++ v2/pkg/reporting/options.go | 3 + v2/pkg/reporting/reporting.go | 13 +++- v2/pkg/types/types.go | 2 + 8 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 v2/pkg/reporting/exporters/jsonl/jsonl.go diff --git a/README.md b/README.md index ea870879c9..5e3571fad6 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,8 @@ OUTPUT: -ms, -matcher-status display match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format - -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans. + -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans + -jle, -jsonl-export string file to export results in JSONL(ine) format as a list of line-delimited JSON objects CONFIGURATIONS: -config string path to the nuclei configuration file diff --git a/README_ID.md b/README_ID.md index 4d044041df..af25cb50ee 100644 --- a/README_ID.md +++ b/README_ID.md @@ -140,7 +140,8 @@ OUTPUT: -ms, -matcher-status display match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format - -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans. + -je, -json-export string file to export results in JSON format as a JSON array. This can be memory intensive in larger scans + -jle, -jsonl-export string file to export results in JSONL(ine) format as a list of line-delimited JSON objects CONFIGURATIONS: -config string path to the nuclei configuration file diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 49ce7d515b..5a81158395 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -181,6 +181,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"), + flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"), ) flagSet.CreateGroup("configs", "Configurations", diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8bf3b3bc41..2e339ed1ad 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -6,7 +6,6 @@ import ( "context" "encoding/json" "fmt" - json_exporter "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" "io" "net/http" _ "net/http/pprof" @@ -50,6 +49,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + json_exporter "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonl" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -338,6 +339,15 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) reportingOptions.JSONExporter = &json_exporter.Options{File: options.JSONExport} } } + if options.JSONLExport != "" { + if reportingOptions != nil { + reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport} + } else { + reportingOptions = &reporting.Options{} + reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport} + } + } + return reportingOptions, nil } diff --git a/v2/pkg/reporting/exporters/jsonl/jsonl.go b/v2/pkg/reporting/exporters/jsonl/jsonl.go new file mode 100644 index 0000000000..76750aaf0a --- /dev/null +++ b/v2/pkg/reporting/exporters/jsonl/jsonl.go @@ -0,0 +1,80 @@ +package jsonl + +import ( + "encoding/json" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "os" + "sync" +) + +type Exporter struct { + options *Options + mutex *sync.Mutex + rows []output.ResultEvent +} + +// Options contains the configuration options for JSONL exporter client +type Options struct { + // File is the file to export found JSONL result to + File string `yaml:"file"` +} + +// New creates a new JSONL exporter integration client based on options. +func New(options *Options) (*Exporter, error) { + exporter := &Exporter{ + mutex: &sync.Mutex{}, + options: options, + rows: []output.ResultEvent{}, + } + return exporter, nil +} + +// Export appends the passed result event to the list of objects to be exported to +// the resulting JSONL file +func (exporter *Exporter) Export(event *output.ResultEvent) error { + exporter.mutex.Lock() + defer exporter.mutex.Unlock() + + // Add the event to the rows + exporter.rows = append(exporter.rows, *event) + + return nil +} + +// Close writes the in-memory data to the JSONL file specified by options.JSONLExport +// and closes the exporter after operation +func (exporter *Exporter) Close() error { + exporter.mutex.Lock() + defer exporter.mutex.Unlock() + + // Open the JSONL file for writing and create it if it doesn't exist + f, err := os.OpenFile(exporter.options.File, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return errors.Wrap(err, "failed to create JSONL file") + } + + // Loop through the rows and convert each to a JSON byte array and write to file + for _, row := range exporter.rows { + // Convert the row to JSON byte array and append a trailing newline. This is treated as a single line in JSONL + obj, err := json.Marshal(row) + if err != nil { + return errors.Wrap(err, "failed to generate row for JSONL report") + } + + // Add a trailing newline to the JSON byte array to confirm with the JSONL format + obj = append(obj, '\n') + + // Attempt to append the JSON line to file specified in options.JSONLExport + if _, err = f.Write(obj); err != nil { + return errors.Wrap(err, "failed to append JSONL line") + } + } + + // Close the file + if err := f.Close(); err != nil { + return errors.Wrap(err, "failed to close JSONL file") + } + + return nil +} diff --git a/v2/pkg/reporting/options.go b/v2/pkg/reporting/options.go index 32fea55a13..579df9de12 100644 --- a/v2/pkg/reporting/options.go +++ b/v2/pkg/reporting/options.go @@ -3,6 +3,7 @@ package reporting import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonl" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/splunk" @@ -34,6 +35,8 @@ type Options struct { SplunkExporter *splunk.Options `yaml:"splunkhec"` // JSONExporter contains configuration options for JSON Exporter Module JSONExporter *jsonexporter.Options `yaml:"json"` + // JSONLExporter contains configuration options for JSONL Exporter Module + JSONLExporter *jsonl.Options `yaml:"jsonl"` HttpClient *retryablehttp.Client `yaml:"-"` } diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index d793549206..23197ad1e9 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -1,10 +1,12 @@ package reporting import ( - json_exporter "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" "os" "path/filepath" + json_exporter "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonl" + "go.uber.org/multierr" "gopkg.in/yaml.v2" @@ -141,6 +143,13 @@ func New(options *Options, db string) (Client, error) { } client.exporters = append(client.exporters, exporter) } + if options.JSONLExporter != nil { + exporter, err := jsonl.New(options.JSONLExporter) + if err != nil { + return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation) + } + client.exporters = append(client.exporters, exporter) + } if options.ElasticsearchExporter != nil { options.ElasticsearchExporter.HttpClient = options.HttpClient exporter, err := es.New(options.ElasticsearchExporter) @@ -189,6 +198,8 @@ func CreateConfigIfNotExists() error { SarifExporter: &sarif.Options{}, ElasticsearchExporter: &es.Options{}, SplunkExporter: &splunk.Options{}, + JSONExporter: &json_exporter.Options{}, + JSONLExporter: &jsonl.Options{}, } reportingFile, err := os.Create(reportingConfig) if err != nil { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index a2cf837e3c..838981e3b3 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -236,6 +236,8 @@ type Options struct { JSONRequests bool // JSONExport is the file to export JSON output format to JSONExport string + // JSONLExport is the file to export JSONL output format to + JSONLExport string // Cloud enables nuclei cloud scan execution Cloud bool // EnableProgressBar enables progress bar From ecfd619b3ea9e51e4067413a1aa90dd38f4d134f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 18:35:27 +0530 Subject: [PATCH 10/34] chore(deps): bump golang from 1.20.2-alpine to 1.20.3-alpine (#3517) Bumps golang from 1.20.2-alpine to 1.20.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 255bf68f18..ec6e03551e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.20.2-alpine AS build-env +FROM golang:1.20.3-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app From 16d72149999eb4b47bcd11cbe95d000441ae4d3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 18:36:12 +0530 Subject: [PATCH 11/34] chore(deps): bump github.com/go-rod/rod from 0.112.4 to 0.112.8 in /v2 (#3522) Bumps [github.com/go-rod/rod](https://github.com/go-rod/rod) from 0.112.4 to 0.112.8. - [Release notes](https://github.com/go-rod/rod/releases) - [Commits](https://github.com/go-rod/rod/compare/v0.112.4...v0.112.8) --- updated-dependencies: - dependency-name: github.com/go-rod/rod dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 087fa26b63..964fdf2c67 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/go-playground/validator/v10 v10.11.2 - github.com/go-rod/rod v0.112.4 + github.com/go-rod/rod v0.112.8 github.com/gobwas/ws v1.1.0 github.com/google/go-github v17.0.0+incompatible github.com/itchyny/gojq v0.12.11 diff --git a/v2/go.sum b/v2/go.sum index 87f8f0a12f..d8d6709373 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -196,8 +196,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= -github.com/go-rod/rod v0.112.4 h1:Ck002nM6rCORdVFtD778WxiadS5oJsmqytjXTG5bqiQ= -github.com/go-rod/rod v0.112.4/go.mod h1:ElViL9ABbcshNQw93+11FrYRH92RRhMKleuILo6+5V0= +github.com/go-rod/rod v0.112.8 h1:lYFnHv/lFyjW/Ye0IhyKLeHw/zfhHbSTqawoCi2z/nI= +github.com/go-rod/rod v0.112.8/go.mod h1:ElViL9ABbcshNQw93+11FrYRH92RRhMKleuILo6+5V0= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= From 216d7b18d3824925dc7cda3f6a73b278fc325d65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Apr 2023 18:55:14 +0530 Subject: [PATCH 12/34] chore(deps): bump github.com/projectdiscovery/tlsx in /v2 (#3521) Bumps [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) from 1.0.6-0.20230328111908-f4528797e948 to 1.0.6. - [Release notes](https://github.com/projectdiscovery/tlsx/releases) - [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/tlsx/commits/v1.0.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/tlsx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 964fdf2c67..7409676c42 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.0.6-0.20230328111908-f4528797e948 + github.com/projectdiscovery/tlsx v1.0.6 github.com/projectdiscovery/uncover v1.0.2 github.com/projectdiscovery/utils v0.0.18 github.com/projectdiscovery/wappalyzergo v0.0.88 diff --git a/v2/go.sum b/v2/go.sum index d8d6709373..924db89974 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -468,8 +468,8 @@ github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGL github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.0.6-0.20230328111908-f4528797e948 h1:bAx0ittZ88QwV0ohxjz4fnhLvNYjYTgXcTlgQP+hzY4= -github.com/projectdiscovery/tlsx v1.0.6-0.20230328111908-f4528797e948/go.mod h1:WVHBfdZ9x2zYxfDdmtr73W3soq/h2koI8eJ/ubroSnI= +github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMKHIrxm8= +github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY= github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= github.com/projectdiscovery/utils v0.0.18 h1:gyBMnA4y2ryui0G98iFqKAXuNdoSy6Z6K0/1KHB0czU= From c4c89177eba1c03dd80ad0e6ea0d37084aeb69d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Apr 2023 18:56:02 +0530 Subject: [PATCH 13/34] chore(deps): bump golang.org/x/text from 0.8.0 to 0.9.0 in /v2 (#3518) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.8.0 to 0.9.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.8.0...v0.9.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 7409676c42..a94c8843d7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -46,7 +46,7 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/net v0.8.0 golang.org/x/oauth2 v0.6.0 - golang.org/x/text v0.8.0 + golang.org/x/text v0.9.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) diff --git a/v2/go.sum b/v2/go.sum index 924db89974..595432b337 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -768,8 +768,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From c060836461e63553519c4cda891949de8e5cfd24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Apr 2023 18:56:23 +0530 Subject: [PATCH 14/34] chore(deps): bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /v2 (#3520) Bumps [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) from 1.11.55 to 1.11.61. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.11.55...feature/s3/manager/v1.11.61) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 32 ++++++++++++------------ v2/go.sum | 75 ++++++++++++++++++++++++------------------------------- 2 files changed, 48 insertions(+), 59 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index a94c8843d7..6ef0115a7b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -55,11 +55,11 @@ require ( github.com/DataDog/gostackparse v0.6.0 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 - github.com/aws/aws-sdk-go-v2 v1.17.7 - github.com/aws/aws-sdk-go-v2/config v1.18.19 - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55 - github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 + github.com/aws/aws-sdk-go-v2 v1.17.8 + github.com/aws/aws-sdk-go-v2/config v1.18.20 + github.com/aws/aws-sdk-go-v2/credentials v1.13.19 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61 + github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2 github.com/docker/go-units v0.5.0 github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.6.1 @@ -92,10 +92,10 @@ require ( github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bits-and-blooms/bitset v1.3.1 // indirect github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect @@ -225,14 +225,14 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/alecthomas/chroma v0.10.0 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.8 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 595432b337..011295cdcc 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -61,53 +61,42 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.17.8 h1:GMupCNNI7FARX27L7GjCJM8NgivWbRgpjNI/hOQjFS8= +github.com/aws/aws-sdk-go-v2 v1.17.8/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= -github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55 h1:ClZKHmu2QIRQCEQ2Y2upfu4JPO0pG69Ce5eiq3PS2V4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55/go.mod h1:L/h5B6I7reig2QJXCGY0e0NVx4hYCcjETmsfR02hFng= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21 h1:QdxdY43AiwsqG/VAqHA7bIVSm3rKr8/p9i05ydA0/RM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21/go.mod h1:QtIEat7ksHH8nFItljyvMI0dGj8lipK2XZ4PhNihTEU= +github.com/aws/aws-sdk-go-v2/config v1.18.20 h1:yYy+onqmLmDVZtx0mkqbx8aJPl+58V6ivLbLDZ2Qztc= +github.com/aws/aws-sdk-go-v2/config v1.18.20/go.mod h1:RWjF39RiDevmHw/+VaD8F0A36OPIPTHQQyRx0eZohnw= +github.com/aws/aws-sdk-go-v2/credentials v1.13.19 h1:FWHJy9uggyQCSEhovtl/6W6rW9P6DSr62GUeY/TS6Eo= +github.com/aws/aws-sdk-go-v2/credentials v1.13.19/go.mod h1:2m4uvLvl5hvQezVkLeBBUGMEDm5GcUNc3016W6d3NGg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 h1:jOzQAesnBFDmz93feqKnsTHsXrlwWORNZMFHMV+WLFU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2/go.mod h1:cDh1p6XkSGSwSRIArWRc6+UqAQ7x4alQ0QfpVR6f+co= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61 h1:0fHTNkoMAz7jbXSyo0SLubbTJEO+AgvkZ0iWT9DbEsk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61/go.mod h1:i8l1At/vjpY8xf1ivKUBJE4+DQyE0gM9Zdg3ZpC/7RU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 h1:dpbVNUjczQ8Ae3QKHbpHBpfvaVkRdesxpTOe9pTouhU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32/go.mod h1:RudqOgadTWdcS3t/erPQo24pcVEoYyqj/kKW5Vya21I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 h1:QH2kOS3Ht7x+u0gHCh06CXL/h6G8LQJFpZfFBYBNboo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26/go.mod h1:vq86l7956VgFr0/FWQ2BWnK07QC3WYsepKzy33qqY5U= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 h1:HbH1VjUgrCdLJ+4lnnuLI4iVNRvBbBELGaJ5f69ClA8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33/go.mod h1:zG2FcwjQarWaqXSCGpgcr3RSjZ6dHGguZSppUL0XR7Q= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 h1:zsg+5ouVLLbePknVZlUMm1ptwyQLkjjLMWnN+kVs5dA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24/go.mod h1:+fFaIjycTmpV6hjmPTbyU9Kp5MI/lA+bbibcAtmlhYA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24 h1:Qmm8klpAdkuN3/rPrIMa/hZQ1z93WMBPjOzdAsbSnlo= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24/go.mod h1:QelGeWBVRh9PbbXsfXKTFlU9FjT6W2yP+dW5jMQzOkg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23 h1:qc+RW0WWZ2KApMnsu/EVCPqLTyIH55uc7YQq7mq4XqE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23/go.mod h1:FJhZWVWBCcgAF8jbep7pxQ1QUsjzTwa9tvEXGw2TDRo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 h1:kFfb+NMap4R7nDvBYyABa/nw7KFMtAfygD1Hyoxh4uE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5/go.mod h1:Dze3kNt4T+Dgb8YCfuIFSBLmE6hadKNxqfdF0Xmqz1I= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27 h1:qIw7Hg5eJEc1uSxg3hRwAthPAO7NeOd4dPxhaTi0yB0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.27/go.mod h1:Zz0kvhcSlu3NX4XJkaGgdjaa+u7a9LYuy8JKxA5v3RM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 h1:uUt4XctZLhl9wBE1L8lobU3bVN8SNUP7T+olb0bWBO4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26/go.mod h1:Bd4C/4PkVGubtNe5iMXu5BNnaBi/9t/UsFspPt4ram8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1 h1:lRWp3bNu5wy0X3a8GS42JvZFlv++AKsMdzEnoiVJrkg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1/go.mod h1:VXBHSxdN46bsJrkniN68psSwbyBKsazQfU2yX/iSDso= +github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2 h1:iOZoYePk+EuBI1tC7bxeRjO+JvClcYm2fZYW5WPIOMQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2/go.mod h1:aSl9/LJltSz1cVusiR/Mu8tvI4Sv/5w/WWrJmmkNii0= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 h1:rrYYhsvcvg6CDDoo4GHKtAWBFutS86CpmGvqHJHYL9w= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.7/go.mod h1:GNIveDnP+aE3jujyUSH5aZ/rktsTM5EvtKnCqBZawdw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 h1:Vjpjt3svuJ/u+eKRfycZwqLsLoxyuvvZyHMJSk+3k58= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7/go.mod h1:44qFP1g7pfd+U+sQHLPalAPKnyfTZjJsYR4xIwsJy5o= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.8 h1:SQ8pPoXfzuz4DImO4KAi9xhO4ANG0Ckb5clZ5GoRAPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.8/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= From 45cc676f964948a5fddfec115a7308e551ca414f Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Wed, 12 Apr 2023 01:50:58 +0530 Subject: [PATCH 15/34] Evaluate payload variables (#3503) * Evaluate payload variables * Add variables evaluation * Extend variables test - to check evaluation of global variables in variables - to check evaluation of golbal variables in payload * Add default and cli variables to websocket, whois and dns proto - use url.Parse with urlutil.Parse --- integration_tests/http/variables.yaml | 11 +++++-- v2/cmd/integration-test/http.go | 2 +- .../common/expressions/expressions.go | 4 +-- .../protocols/common/variables/variables.go | 20 ++++++++++++ v2/pkg/protocols/dns/request.go | 5 ++- v2/pkg/protocols/http/build_request.go | 15 ++++++++- v2/pkg/protocols/http/request.go | 6 ---- v2/pkg/protocols/network/request.go | 6 ++-- v2/pkg/protocols/websocket/websocket.go | 32 +++++++++++-------- v2/pkg/protocols/whois/whois.go | 10 ++++-- 10 files changed, 79 insertions(+), 32 deletions(-) diff --git a/integration_tests/http/variables.yaml b/integration_tests/http/variables.yaml index ad3e3348d0..59562b7fdc 100644 --- a/integration_tests/http/variables.yaml +++ b/integration_tests/http/variables.yaml @@ -7,7 +7,7 @@ info: variables: a1: "value" - a2: "{{base64('hello')}}" + a2: "{{base64('{{Host}}')}}" requests: - raw: @@ -16,11 +16,16 @@ requests: Host: {{FQDN}} Test: {{a1}} Another: {{a2}} - + Email: {{ username }} + payloads: + username: + - jon.doe@{{ FQDN }} stop-at-first-match: true matchers-condition: or matchers: - type: word + condition: and words: - "value" - - "aGVsbG8=" \ No newline at end of file + - "MTI3LjAuMC4x" # 127.0.0.1 + - "jon.doe@127.0.0.1" diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 2d339af46c..545459cca7 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -1050,7 +1050,7 @@ type httpVariables struct{} func (h *httpVariables) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - fmt.Fprintf(w, "%s\n%s", r.Header.Get("Test"), r.Header.Get("Another")) + fmt.Fprintf(w, "%s\n%s\n%s", r.Header.Get("Test"), r.Header.Get("Another"), r.Header.Get("Email")) }) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 2bd6228cf0..870f75aaeb 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -40,7 +40,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) { // - simple: containing base values keys (variables) // - complex: containing helper functions [ + variables] // literals like {{2+2}} are not considered expressions - expressions := findExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base) + expressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base) for _, expression := range expressions { // replace variable placeholders with base values expression = replacer.Replace(expression, base) @@ -62,7 +62,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) { // maxIterations to avoid infinite loop const maxIterations = 250 -func findExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string { +func FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string { var ( iterations int exps []string diff --git a/v2/pkg/protocols/common/variables/variables.go b/v2/pkg/protocols/common/variables/variables.go index 2ab94569fa..32f174cfcb 100644 --- a/v2/pkg/protocols/common/variables/variables.go +++ b/v2/pkg/protocols/common/variables/variables.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) @@ -15,6 +16,7 @@ import ( // Variable is a key-value pair of strings that can be used // throughout template. type Variable struct { + LazyEval bool `yaml:"-" json:"-"` // LazyEval is used to evaluate variables lazily if it using any expression or global variables utils.InsertionOrderedStringMap `yaml:"-" json:"-"` } @@ -32,6 +34,11 @@ func (variables *Variable) UnmarshalYAML(unmarshal func(interface{}) error) erro if err := unmarshal(&variables.InsertionOrderedStringMap); err != nil { return err } + + if variables.checkForLazyEval() { + return nil + } + evaluated := variables.Evaluate(map[string]interface{}{}) for k, v := range evaluated { @@ -87,3 +94,16 @@ func evaluateVariableValue(expression string, values, processing map[string]inte return result } + +// checkForLazyEval checks if the variables have any lazy evaluation i.e any dsl function +// and sets the flag accordingly. +func (variables *Variable) checkForLazyEval() bool { + variables.ForEach(func(key string, value interface{}) { + ret := expressions.FindExpressions(types.ToString(value), marker.ParenthesisOpen, marker.ParenthesisClose, nil) + if len(ret) > 0 { + variables.LazyEval = true + return + } + }) + return variables.LazyEval +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index a3060dcf61..5574000754 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -45,9 +45,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, if err != nil { return errors.Wrap(err, "could not build request") } + vars := GenerateVariables(domain) + // optionvars are vars passed from CLI or env variables + optionVars := generators.BuildPayloadFromOptions(request.options.Options) // merge with metadata (eg. from workflow context) - vars = generators.MergeMaps(vars, metadata) + vars = generators.MergeMaps(vars, metadata, optionVars) variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(variablesMap, vars) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 3d1af6445f..ae9b9e3f26 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -104,9 +104,22 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, defaultReqVars := utils.GenerateVariablesWithURL(parsed, hasTrailingSlash, contextargs.GenerateVariables(input)) // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(r.request.options.Options) + + variablesMap, interactURLs := r.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(defaultReqVars, optionVars), r.options.Interactsh) + if len(interactURLs) > 0 { + r.interactshURLs = append(r.interactshURLs, interactURLs...) + } // allVars contains all variables from all sources - allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars) + allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars, variablesMap) + // Evaluate payload variables + // eg: payload variables can be username: jon.doe@{{Hostname}} + for payloadName, payloadValue := range payloads { + payloads[payloadName], err = expressions.Evaluate(types.ToString(payloadValue), allVars) + if err != nil { + return nil, ErrEvalExpression.Wrap(err).WithTag("http") + } + } // finalVars contains allVars and any generator/fuzzing specific payloads // payloads used in generator should be given the most preference finalVars := generators.MergeMaps(allVars, payloads) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d000ffc8da..cf2b563fe4 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -345,8 +345,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // returns two values, error and skip, which skips the execution for the request instance. executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) { hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) - variablesMap, interactURLs := request.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(dynamicValues, payloads), request.options.Interactsh) - dynamicValue = generators.MergeMaps(variablesMap, dynamicValue) request.options.RateLimiter.Take() @@ -366,10 +364,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa defer generatedHttpRequest.customCancelFunction() } - // If the variables contain interactsh urls, use them - if len(interactURLs) > 0 { - generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...) - } hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0 if input.MetaInput.Input == "" { input.MetaInput.Input = generatedHttpRequest.URL() diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 7e7ed39dfb..7dc706d6b6 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -51,11 +51,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } + variables := generateNetworkVariables(address) + variablesMap := request.options.Variables.Evaluate(variables) + variables = generators.MergeMaps(variablesMap, variables) for _, kv := range request.addresses { - variables := generateNetworkVariables(address) - variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(variables, variables)) - variables = generators.MergeMaps(variablesMap, variables) actualAddress := replacer.Replace(kv.address, variables) if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 7f65341dda..324194043f 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -168,22 +168,14 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { header := http.Header{} - payloadValues := make(map[string]interface{}) - for k, v := range dynamicValues { - payloadValues[k] = v - } - parsed, err := url.Parse(input) + parsed, err := urlutil.Parse(input) if err != nil { return errors.Wrap(err, parseUrlErrorMessage) } - payloadValues["Hostname"] = parsed.Host - payloadValues["Host"] = parsed.Hostname() - payloadValues["Scheme"] = parsed.Scheme - requestPath := parsed.Path - if values := urlutil.GetParams(parsed.Query()); len(values) > 0 { - requestPath = requestPath + "?" + values.Encode() - } - payloadValues["Path"] = requestPath + defaultVars := getWebsocketVariables(parsed) + optionVars := generators.BuildPayloadFromOptions(request.options.Options) + variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues)) + payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues) requestOptions := request.options for key, value := range request.Headers { @@ -415,3 +407,17 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent func (request *Request) Type() templateTypes.ProtocolType { return templateTypes.WebsocketProtocol } + +func getWebsocketVariables(input *urlutil.URL) map[string]interface{} { + websocketVariables := make(map[string]interface{}) + + websocketVariables["Hostname"] = input.Host + websocketVariables["Host"] = input.Hostname() + websocketVariables["Scheme"] = input.Scheme + requestPath := input.Path + if values := urlutil.GetParams(input.URL.Query()); len(values) > 0 { + requestPath = requestPath + "?" + values.Encode() + } + websocketVariables["Path"] = requestPath + return websocketVariables +} diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index ddabfc0d9d..c457eb5c4e 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" @@ -23,6 +24,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois/rdapclientpool" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" + urlutil "github.com/projectdiscovery/utils/url" ) // Request is a request for the WHOIS protocol @@ -85,7 +87,11 @@ func (request *Request) GetID() string { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // generate variables - variables := generateVariables(input.MetaInput.Input) + defaultVars := generateVariables(input.MetaInput.Input) + optionVars := generators.BuildPayloadFromOptions(request.options.Options) + vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues)) + + variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues) if vardump.EnableVarDump { gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) @@ -183,7 +189,7 @@ func (request *Request) Type() templateTypes.ProtocolType { func generateVariables(input string) map[string]interface{} { var domain string - parsed, err := url.Parse(input) + parsed, err := urlutil.Parse(input) if err != nil { return map[string]interface{}{"Input": input} } From ece20ec15ce7faf63f6b40635df5d3da3fe79e54 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Wed, 12 Apr 2023 12:49:58 +0530 Subject: [PATCH 16/34] use proxyutils + fix proxy panic (#3526) --- v2/go.mod | 4 +- v2/go.sum | 10 ++-- v2/internal/runner/proxy.go | 92 ++++++------------------------------- v2/pkg/types/proxy.go | 3 -- 4 files changed, 22 insertions(+), 87 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 6ef0115a7b..350d42a1a9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -80,7 +80,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.0.6 github.com/projectdiscovery/uncover v1.0.2 - github.com/projectdiscovery/utils v0.0.18 + github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 github.com/projectdiscovery/wappalyzergo v0.0.88 github.com/stretchr/testify v1.8.2 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -211,7 +211,7 @@ require ( golang.org/x/crypto v0.7.0 golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 golang.org/x/mod v0.9.0 // indirect - golang.org/x/sys v0.6.0 // indirect + golang.org/x/sys v0.7.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 011295cdcc..4f56a0f01f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -461,8 +461,10 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY= github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= -github.com/projectdiscovery/utils v0.0.18 h1:gyBMnA4y2ryui0G98iFqKAXuNdoSy6Z6K0/1KHB0czU= -github.com/projectdiscovery/utils v0.0.18/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs= +github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707 h1:usIIWZ/didvXCAhJ39ruuUKH1po4gTV7O097B+YGXDM= +github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o= +github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 h1:UBOE9Eob1wj7YZ1MGBbtHvc3ptqBHvXxNKRVxTg21Rc= +github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o= github.com/projectdiscovery/wappalyzergo v0.0.88 h1:N/1vFlKmc3GJco9rANJdHrxg8jdav/xmnICo8rztmH8= github.com/projectdiscovery/wappalyzergo v0.0.88/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -738,8 +740,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index 18e65fd898..5bf464b7a9 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -2,30 +2,26 @@ package runner import ( "bufio" - "errors" "fmt" - "net" "net/url" "os" "strings" - "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + proxyutils "github.com/projectdiscovery/utils/proxy" ) -var proxyURLList []url.URL - // loadProxyServers load list of proxy servers from file or comma seperated func loadProxyServers(options *types.Options) error { if len(options.Proxy) == 0 { return nil } + proxyList := []string{} for _, p := range options.Proxy { - if proxyURL, err := validateProxyURL(p); err == nil { - proxyURLList = append(proxyURLList, proxyURL) - } else if fileutil.FileExists(p) { + if fileutil.FileExists(p) { file, err := os.Open(p) if err != nil { return fmt.Errorf("could not open proxy file: %w", err) @@ -37,91 +33,31 @@ func loadProxyServers(options *types.Options) error { if strings.TrimSpace(proxy) == "" { continue } - if proxyURL, err := validateProxyURL(proxy); err != nil { - return err - } else { - proxyURLList = append(proxyURLList, proxyURL) - } + proxyList = append(proxyList, proxy) } } else { - return fmt.Errorf("invalid proxy file or URL provided for %s", p) - } - } - return processProxyList(options) -} - -func processProxyList(options *types.Options) error { - if len(proxyURLList) == 0 { - return fmt.Errorf("could not find any valid proxy") - } else { - done := make(chan bool) - exitCounter := make(chan bool) - counter := 0 - for _, url := range proxyURLList { - go runProxyConnectivity(url, options, done, exitCounter) - } - for { - select { - case <-done: - { - close(done) - return nil - } - case <-exitCounter: - { - if counter += 1; counter == len(proxyURLList) { - return errors.New("no reachable proxy found") - } - } - } - } - } -} - -func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { - if err := testProxyConnection(proxyURL, options.Timeout); err == nil { - if types.ProxyURL == "" && types.ProxySocksURL == "" { - assignProxyURL(proxyURL, options) - done <- true + proxyList = append(proxyList, p) } - } else { - gologger.Debug().Msgf("Proxy validation failed for '%s': %s", proxyURL.String(), err) } - exitCounter <- true -} - -func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { - timeout := time.Duration(timeoutDelay) * time.Second - _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) + aliveProxy, err := proxyutils.GetAnyAliveProxy(options.Timeout, proxyList...) if err != nil { return err } - return nil -} - -func assignProxyURL(proxyURL url.URL, options *types.Options) { + proxyURL, err := url.Parse(aliveProxy) + if err != nil { + return errorutil.WrapfWithNil(err, "failed to parse proxy got %v", err) + } if options.ProxyInternal { os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) } - if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { + if proxyURL.Scheme == proxyutils.HTTP || proxyURL.Scheme == proxyutils.HTTPS { types.ProxyURL = proxyURL.String() types.ProxySocksURL = "" gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) - } else if proxyURL.Scheme == types.SOCKS5 { + } else if proxyURL.Scheme == proxyutils.SOCKS5 { types.ProxyURL = "" types.ProxySocksURL = proxyURL.String() gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) } -} - -func validateProxyURL(proxy string) (url.URL, error) { - if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { - return *url, nil - } - return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") -} - -// isSupportedProtocol checks given protocols are supported -func isSupportedProtocol(value string) bool { - return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 + return nil } diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go index a45b4eca67..1db9ac3c00 100644 --- a/v2/pkg/types/proxy.go +++ b/v2/pkg/types/proxy.go @@ -2,9 +2,6 @@ package types const ( HTTP_PROXY_ENV = "HTTP_PROXY" - SOCKS5 = "socks5" - HTTP = "http" - HTTPS = "https" ) var ( From 70297413381421eff9386c1b79393786cd3bfa69 Mon Sep 17 00:00:00 2001 From: sduc Date: Wed, 12 Apr 2023 12:49:49 +0200 Subject: [PATCH 17/34] [DRAFT] Annotate CVE: add configurations metadata (#3486) * improve annotate to use new nvd apis * annotate * improvements * fix mod files * fetch EPSS and only write CPE * lint fixes --------- Co-authored-by: Mzack9999 Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- v2/cmd/cve-annotate/main.go | 132 ++++++++++++++++++++++++++++++++---- v2/go.mod | 2 +- v2/go.sum | 14 ++-- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index 2c17030c1e..4aa7c9ec68 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -5,10 +5,13 @@ import ( "encoding/json" "flag" "fmt" + "io" "log" + "net/http" "net/url" "os" "regexp" + "strconv" "strings" "github.com/pkg/errors" @@ -56,10 +59,7 @@ func process() error { } defer os.RemoveAll(tempDir) - client, err := nvd.NewClient(tempDir) - if err != nil { - return err - } + client := nvd.NewClientV2() catalog := disk.NewCatalog(*templateDir) paths, err := catalog.GetTemplatePath(*input) @@ -109,7 +109,7 @@ var badRefs = []string{ "secunia.com/", } -func getCVEData(client *nvd.Client, filePath, data string) { +func getCVEData(client *nvd.ClientV2, filePath, data string) { matches := idRegex.FindAllStringSubmatch(data, 1) if len(matches) == 0 { return @@ -142,13 +142,18 @@ func getCVEData(client *nvd.Client, filePath, data string) { return } var cweID []string - for _, problemData := range cveItem.CVE.Problemtype.ProblemtypeData { - for _, description := range problemData.Description { + for _, weaknessData := range cveItem.Cve.Weaknesses { + for _, description := range weaknessData.Description { cweID = append(cweID, description.Value) } } - cvssScore := cveItem.Impact.BaseMetricV3.CvssV3.BaseScore - cvssMetrics := cveItem.Impact.BaseMetricV3.CvssV3.VectorString + cvssData, err := getPrimaryCVSSData(cveItem) + if err != nil { + log.Printf("Could not get CVSS data %s: %s\n", cveName, err) + return + } + cvssScore := cvssData.BaseScore + cvssMetrics := cvssData.VectorString // Perform some hacky string replacement to place the metadata in templates infoBlockIndexData := data[strings.Index(data, "info:"):] @@ -191,12 +196,13 @@ func getCVEData(client *nvd.Client, filePath, data string) { } } // If there is no description field, fill the description from CVE information - hasDescriptionData := len(cveItem.CVE.Description.DescriptionData) > 0 + enDescription, err := getEnglishLangString(cveItem.Cve.Descriptions) + hasDescriptionData := err != nil isDescriptionEmpty := infoBlock.Info.Description == "" if isDescriptionEmpty && hasDescriptionData { changed = true // removes all new lines - description := stringsutil.ReplaceAll(cveItem.CVE.Description.DescriptionData[0].Value, "", "\n", "\\", "'", "\t") + description := stringsutil.ReplaceAll(enDescription, "", "\n", "\\", "'", "\t") description += "\n" infoBlock.Info.Description = description } @@ -205,13 +211,13 @@ func getCVEData(client *nvd.Client, filePath, data string) { var referenceDataURLs []string // skip sites that are no longer alive - for _, reference := range cveItem.CVE.References.ReferenceData { + for _, reference := range cveItem.Cve.References { if stringsutil.ContainsAny(reference.URL, badRefs...) { continue } referenceDataURLs = append(referenceDataURLs, reference.URL) } - hasReferenceData := len(cveItem.CVE.References.ReferenceData) > 0 + hasReferenceData := len(cveItem.Cve.References) > 0 areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs) referencesCount := len(infoBlock.Info.Reference) if hasReferenceData && !areCveReferencesContained { @@ -226,6 +232,36 @@ func getCVEData(client *nvd.Client, filePath, data string) { infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference)) } + cpeSet := map[string]bool{} + for _, config := range cveItem.Cve.Configurations { + // Right now this covers only simple configurations. More complex configurations can have multiple CPEs + if len(config.Nodes) == 1 { + changed = true + node := config.Nodes[0] + for _, match := range node.CpeMatch { + cpeSet[extractVersionlessCpe((match.Criteria))] = true + } + } + } + uniqueCpes := make([]string, 0, len(cpeSet)) + for k := range cpeSet { + uniqueCpes = append(uniqueCpes, k) + } + if len(uniqueCpes) == 1 { + infoBlock.Info.Classification.Cpe = uniqueCpes[0] + } + + epss, err := fetchEpss(cveName) + if err != nil { + log.Printf("Could not fetch Epss score: %s\n", err) + return + } + hasEpssChanged := epss != infoBlock.Info.Classification.EpssScore + if hasEpssChanged { + changed = true + infoBlock.Info.Classification.EpssScore = epss + } + var newInfoBlock bytes.Buffer yamlEncoder := yaml.NewEncoder(&newInfoBlock) yamlEncoder.SetIndent(yamlIndentSpaces) @@ -243,6 +279,29 @@ func getCVEData(client *nvd.Client, filePath, data string) { } } +func getPrimaryCVSSData(vuln nvd.Vulnerability) (nvd.CvssData, error) { + for _, data := range vuln.Cve.Metrics.CvssMetricV31 { + if data.Type == "Primary" { + return data.CvssData, nil + } + } + for _, data := range vuln.Cve.Metrics.CvssMetricV3 { + if data.Type == "Primary" { + return data.CvssData, nil + } + } + return nvd.CvssData{}, fmt.Errorf("no primary cvss metric found") +} + +func getEnglishLangString(data []nvd.LangString) (string, error) { + for _, item := range data { + if item.Lang == "en" { + return item.Value, nil + } + } + return "", fmt.Errorf("no english item found") +} + func isSeverityMatchingCvssScore(severity string, score float64) string { if score == 0.0 { return "" @@ -264,6 +323,51 @@ func isSeverityMatchingCvssScore(severity string, score float64) string { return "" } +func extractVersionlessCpe(cpe string) string { + parts := strings.Split(cpe, ":") + versionlessPart := parts[0:5] + rest := strings.Split(strings.Repeat("*", len(parts)-len(versionlessPart)), "") + return strings.Join(append(versionlessPart, rest...), ":") +} + +type ApiFirstEpssResponse struct { + Status string `json:"status"` + StatusCode int `json:"status-code"` + Version string `json:"version"` + Access string `json:"access"` + Total int `json:"total"` + Offset int `json:"offset"` + Limit int `json:"limit"` + Data []struct { + Cve string `json:"cve"` + Epss string `json:"epss"` + Percentile string `json:"percentile"` + Date string `json:"date"` + } `json:"data"` +} + +func fetchEpss(cveId string) (float64, error) { + resp, err := http.Get(fmt.Sprintf("https://api.first.org/data/v1/epss?cve=%s", cveId)) + if err != nil { + return 0, fmt.Errorf("unable to fetch EPSS data from first.org: %v", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, fmt.Errorf("unable to read reponse body: %v", err) + } + var parsedResp ApiFirstEpssResponse + err = json.Unmarshal(body, &parsedResp) + if err != nil { + return 0, fmt.Errorf("error while parsing EPSS response: %v", err) + } + if len(parsedResp.Data) != 1 { + return 0, fmt.Errorf("unexpected number of results in EPSS response. Expecting exactly 1, got %v", len(parsedResp.Data)) + } + epss := parsedResp.Data[0].Epss + return strconv.ParseFloat(epss, 64) +} + type cisaKEVData struct { Vulnerabilities []struct { CVEID string `json:"cveID"` @@ -392,6 +496,8 @@ type TemplateClassification struct { CvssScore float64 `yaml:"cvss-score,omitempty"` CveId string `yaml:"cve-id,omitempty"` CweId string `yaml:"cwe-id,omitempty"` + Cpe string `yaml:"cpe,omitempty"` + EpssScore float64 `yaml:"epss-score,omitempty"` } type TemplateInfo struct { diff --git a/v2/go.mod b/v2/go.mod index 350d42a1a9..469bd19586 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -74,7 +74,7 @@ require ( github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/httpx v1.2.9 github.com/projectdiscovery/mapcidr v1.1.1 - github.com/projectdiscovery/nvd v1.0.9 + github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 diff --git a/v2/go.sum b/v2/go.sum index 4f56a0f01f..f0c6d4779c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -58,6 +58,7 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -362,6 +363,7 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= @@ -441,8 +443,8 @@ github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e github.com/projectdiscovery/mapcidr v1.1.1/go.mod h1:yyp9ghqmmC0+r5DySgDBXE4cf2QW8SBloVESCteWiAg= github.com/projectdiscovery/networkpolicy v0.0.4 h1:zcGjEqZbyECZEdyCy1jVuwOS7Ww1mzgCefQU75XqdJA= github.com/projectdiscovery/networkpolicy v0.0.4/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w= -github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE= -github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og= +github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 h1:aDq18tNWbnN5ZM0ADQb+8KB4DEPIGZMXdDmcXyFUoNg= +github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8/go.mod h1:JiVXOIewstCBMPsO+ZnmI43UXMPJGEE1jwuFVz4ujKM= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/rawhttp v0.1.11 h1:NbfunXIVdmFWAhZ864fx09sQLnHOVTYKhAe9P2Cnass= @@ -461,8 +463,7 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY= github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= -github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707 h1:usIIWZ/didvXCAhJ39ruuUKH1po4gTV7O097B+YGXDM= -github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o= +github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 h1:UBOE9Eob1wj7YZ1MGBbtHvc3ptqBHvXxNKRVxTg21Rc= github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o= github.com/projectdiscovery/wappalyzergo v0.0.88 h1:N/1vFlKmc3GJco9rANJdHrxg8jdav/xmnICo8rztmH8= @@ -616,6 +617,7 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= @@ -626,6 +628,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= @@ -654,6 +657,7 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -677,6 +681,7 @@ golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -770,6 +775,7 @@ golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= From 6ffdfcf19cd8a45d24974259ce7b954f00d571bc Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Wed, 12 Apr 2023 23:55:21 +0530 Subject: [PATCH 18/34] fix rate-limit on query fuzzing (#3458) (#3532) --- v2/pkg/protocols/http/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index cf2b563fe4..0e80eca0c2 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -239,7 +239,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) { return false } - + request.options.RateLimiter.Take() req := &generatedRequest{ request: gr.Request, dynamicValues: gr.DynamicValues, From 8ffbebb5ea44280463c86487eb548ba34d6fe18e Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Thu, 13 Apr 2023 08:36:54 -0400 Subject: [PATCH 19/34] Correct spelling in "access" and add arg types (#3536) --- v2/pkg/external/customtemplates/s3.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/external/customtemplates/s3.go b/v2/pkg/external/customtemplates/s3.go index e2da7b15cf..cd1683b0aa 100644 --- a/v2/pkg/external/customtemplates/s3.go +++ b/v2/pkg/external/customtemplates/s3.go @@ -76,8 +76,8 @@ func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key return err } -func getS3Client(ctx context.Context, acccessKey, secretKey, region string) (*s3.Client, error) { - cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(acccessKey, secretKey, "")), config.WithRegion(region)) +func getS3Client(ctx context.Context, accessKey string, secretKey string, region string) (*s3.Client, error) { + cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")), config.WithRegion(region)) if err != nil { return nil, err } From bca0c53a41e2d8cd58779369eb948dacfec360d7 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 13 Apr 2023 15:41:04 +0300 Subject: [PATCH 20/34] Enable --nc option if OS is windows (#3539) Co-authored-by: Mzack9999 Co-authored-by: Sandeep Singh --- v2/cmd/nuclei/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 5a81158395..60f2d6b8f1 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -171,7 +171,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"), flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), - flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isNoColorEnabled(), "disable output content coloring (ANSI escape codes)"), flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"), @@ -379,6 +379,10 @@ on extensive configurability, massive extensibility and ease of use.`) return flagSet } +func isNoColorEnabled() bool { + return runtime.GOOS == "windows" +} + func cleanupOldResumeFiles() { root, err := config.GetConfigDir() if err != nil { From 307085ef4c5741dbb5eee6f61f3b8a3787ebdfb9 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 16 Apr 2023 22:23:51 +0530 Subject: [PATCH 21/34] tlsx dep update (#3554) --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 469bd19586..8876bf025f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -78,7 +78,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.0.6 + github.com/projectdiscovery/tlsx v1.0.7 github.com/projectdiscovery/uncover v1.0.2 github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 github.com/projectdiscovery/wappalyzergo v0.0.88 diff --git a/v2/go.sum b/v2/go.sum index f0c6d4779c..76e352aa8b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -459,8 +459,8 @@ github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGL github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMKHIrxm8= -github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY= +github.com/projectdiscovery/tlsx v1.0.7 h1:qlA9kUhOGRZBvdIVIKjsOAop5pkZiF05lHXrhP0YTXI= +github.com/projectdiscovery/tlsx v1.0.7/go.mod h1:bMz1JMJf1sSBLpk7Y6vLwRZrijxPUvPH9gBTkj6ywwc= github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= From 6f4b1ae48ad37061ee6dc908a5136ec1b100bf08 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 16 Apr 2023 19:49:35 +0200 Subject: [PATCH 22/34] Replacing ccache with generic gcache (#3523) * Replacing ccache with generic gcache * fixing lint issues * removing unecessary hashing + using errorutils * making test more tolerant * removing dead code + refactor * removing redundant code * removing race * maint * moving code * adding more iterations * note + typo * temporary fixing stop-at-first-match with interact * wrapping internal map with mux * sort before running integration test * fix deadlock in requestShouldStopAtFirstMatch * add timeout to integration_test workflow * attempting to remove outer lock * adds interactsh protocol tests in integration_test --------- Co-authored-by: Tarun Koyalwar --- .github/workflows/build-test.yml | 1 + .../http/interactsh-stop-at-first-match.yaml | 12 +- v2/cmd/integration-test/code.go | 2 +- v2/cmd/integration-test/http.go | 4 +- v2/cmd/integration-test/integration-test.go | 68 +++- v2/cmd/integration-test/interactsh.go | 10 + v2/examples/simple.go | 2 +- v2/go.mod | 3 +- v2/go.sum | 8 +- v2/internal/runner/runner.go | 4 +- v2/pkg/output/output.go | 7 + v2/pkg/protocols/common/interactsh/const.go | 18 ++ .../protocols/common/interactsh/interactsh.go | 305 +++++++----------- v2/pkg/protocols/common/interactsh/options.go | 67 ++++ v2/pkg/protocols/http/build_request_test.go | 2 +- v2/pkg/protocols/http/request.go | 42 +-- v2/pkg/utils/atomcache/atomcache.go | 62 ---- 17 files changed, 322 insertions(+), 295 deletions(-) create mode 100644 v2/cmd/integration-test/interactsh.go create mode 100644 v2/pkg/protocols/common/interactsh/const.go create mode 100644 v2/pkg/protocols/common/interactsh/options.go delete mode 100644 v2/pkg/utils/atomcache/atomcache.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ee969182e6..644aa06743 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -42,6 +42,7 @@ jobs: working-directory: v2/ - name: Integration Tests + timeout-minutes: 50 env: GH_ACTION: true GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/integration_tests/http/interactsh-stop-at-first-match.yaml b/integration_tests/http/interactsh-stop-at-first-match.yaml index 415a634ea7..b8063aadb0 100644 --- a/integration_tests/http/interactsh-stop-at-first-match.yaml +++ b/integration_tests/http/interactsh-stop-at-first-match.yaml @@ -8,9 +8,15 @@ info: requests: - method: GET path: - - "{{BaseURL}}" - - "{{BaseURL}}" - - "{{BaseURL}}" + - "{{BaseURL}}/?a=1" + - "{{BaseURL}}/?a=2" + - "{{BaseURL}}/?a=3" + - "{{BaseURL}}/?a=4" + - "{{BaseURL}}/?a=5" + - "{{BaseURL}}/?a=6" + - "{{BaseURL}}/?a=7" + - "{{BaseURL}}/?a=8" + - "{{BaseURL}}/?a=9" headers: url: 'http://{{interactsh-url}}' diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index c6dd5f4f91..eb53be5c69 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -88,7 +88,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { defaultOpts.Templates = goflags.StringSlice{templatePath} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags - interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress) + interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) interactClient, err := interactsh.New(interactOpts) if err != nil { return nil, errors.Wrap(err, "could not create interact client") diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 545459cca7..6a1997b1fc 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -47,8 +47,6 @@ var httpTestcases = map[string]testutils.TestCase{ "http/http-paths.yaml": &httpPaths{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, - "http/interactsh.yaml": &httpInteractshRequest{}, - "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, "http/self-contained.yaml": &httpRequestSelfContained{}, "http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, @@ -71,7 +69,6 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-without-scheme.yaml": &httpGetWithoutScheme{}, "http/cl-body-without-header.yaml": &httpCLBodyWithoutHeader{}, "http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, - "http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{}, } type httpInteractshRequest struct{} @@ -164,6 +161,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error { if err != nil { return err } + // polling is asyncronous, so the interactions may be retrieved after the first request return expectResultsCount(results, 1) } diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 3d0411decf..73ee2a7652 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "sort" "strings" "github.com/logrusorgru/aurora" @@ -22,6 +23,7 @@ var ( protocolTests = map[string]map[string]testutils.TestCase{ "http": httpTestcases, + "interactsh": interactshTestCases, "network": networkTestcases, "dns": dnsTestCases, "workflow": workflowTestcases, @@ -40,9 +42,10 @@ var ( } // For debug purposes - runProtocol = "" - runTemplate = "" - extraArgs = []string{} + runProtocol = "" + runTemplate = "" + extraArgs = []string{} + interactshRetryCount = 3 ) func main() { @@ -58,7 +61,6 @@ func main() { } if runProtocol != "" { - debug = true debugTests() os.Exit(1) } @@ -79,13 +81,36 @@ func main() { } } +// execute a testcase with retry and consider best of N +// intended for flaky tests like interactsh +func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) { + var err error + for i := 0; i < retryCount; i++ { + err = testCase.Execute(templatePath) + if err == nil { + fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath) + return "", nil + } + } + _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed after %v attempts : %s\n", failed, templatePath, retryCount, err) + return templatePath, err +} + func debugTests() { - for tpath, testcase := range protocolTests[runProtocol] { + keys := getMapKeys(protocolTests[runProtocol]) + for _, tpath := range keys { + testcase := protocolTests[runProtocol][tpath] if runTemplate != "" && !strings.Contains(tpath, runTemplate) { continue } - if err := testcase.Execute(tpath); err != nil { - fmt.Printf("\n%v", err.Error()) + if runProtocol == "interactsh" { + if _, err := executeWithRetry(testcase, tpath, interactshRetryCount); err != nil { + fmt.Printf("\n%v", err.Error()) + } + } else { + if _, err := execute(testcase, tpath); err != nil { + fmt.Printf("\n%v", err.Error()) + } } } } @@ -97,10 +122,19 @@ func runTests(customTemplatePaths []string) []string { if len(customTemplatePaths) == 0 { fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto)) } + keys := getMapKeys(testCases) - for templatePath, testCase := range testCases { + for _, templatePath := range keys { + testCase := testCases[templatePath] if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, templatePath) { - if failedTemplatePath, err := execute(testCase, templatePath); err != nil { + var failedTemplatePath string + var err error + if proto == "interactsh" { + failedTemplatePath, err = executeWithRetry(testCase, templatePath, interactshRetryCount) + } else { + failedTemplatePath, err = execute(testCase, templatePath) + } + if err != nil { failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath) } } @@ -120,9 +154,10 @@ func execute(testCase testutils.TestCase, templatePath string) (string, error) { return "", nil } -func expectResultsCount(results []string, expectedNumber int) error { - if len(results) != expectedNumber { - return fmt.Errorf("incorrect number of results: %d (actual) vs %d (expected) \nResults:\n\t%s\n", len(results), expectedNumber, strings.Join(results, "\n\t")) +func expectResultsCount(results []string, expectedNumbers ...int) error { + match := sliceutil.Contains(expectedNumbers, len(results)) + if !match { + return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) } return nil } @@ -132,3 +167,12 @@ func normalizeSplit(str string) []string { return r == ',' }) } + +func getMapKeys[T any](testcases map[string]T) []string { + keys := make([]string, 0, len(testcases)) + for k := range testcases { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/v2/cmd/integration-test/interactsh.go b/v2/cmd/integration-test/interactsh.go new file mode 100644 index 0000000000..035f844d07 --- /dev/null +++ b/v2/cmd/integration-test/interactsh.go @@ -0,0 +1,10 @@ +package main + +import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + +// All Interactsh related testcases +var interactshTestCases = map[string]testutils.TestCase{ + "http/interactsh.yaml": &httpInteractshRequest{}, + "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, + "http/default-matcher-condition.yaml": &httpDefaultMatcherCondition{}, +} diff --git a/v2/examples/simple.go b/v2/examples/simple.go index 129561fa30..5b80d22821 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -50,7 +50,7 @@ func main() { defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags - interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress) + interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) interactClient, err := interactsh.New(interactOpts) if err != nil { log.Fatalf("Could not create interact client: %s\n", err) diff --git a/v2/go.mod b/v2/go.mod index 8876bf025f..4c0d3f6ea6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -18,7 +18,6 @@ require ( github.com/itchyny/gojq v0.12.11 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 - github.com/karlseguin/ccache v2.0.3+incompatible github.com/logrusorgru/aurora v2.0.3+incompatible github.com/miekg/dns v1.1.53 github.com/olekukonko/tablewriter v0.0.5 @@ -53,6 +52,7 @@ require ( require ( github.com/DataDog/gostackparse v0.6.0 + github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go-v2 v1.17.8 @@ -111,7 +111,6 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/karlseguin/expect v1.0.8 // indirect github.com/kataras/jwt v0.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect diff --git a/v2/go.sum b/v2/go.sum index 76e352aa8b..e88a6ba176 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -10,6 +10,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo= @@ -290,11 +292,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= -github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= -github.com/karlseguin/expect v1.0.8 h1:Bb0H6IgBWQpadY25UDNkYPDB9ITqK1xnSoZfAq362fw= -github.com/karlseguin/expect v1.0.8/go.mod h1:lXdI8iGiQhmzpnnmU/EGA60vqKs8NbRNFnhhrJGoD5g= github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -582,8 +580,6 @@ github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2Z github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= -github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= -github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk= github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 2e339ed1ad..ba467d159a 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -262,14 +262,14 @@ func New(options *types.Options) (*Runner, error) { } runner.resumeCfg = resumeCfg - opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) + opts := interactsh.DefaultOptions(runner.output, runner.issuesClient, runner.progress) opts.Debug = runner.options.Debug opts.NoColor = runner.options.NoColor if options.InteractshURL != "" { opts.ServerURL = options.InteractshURL } opts.Authorization = options.InteractshToken - opts.CacheSize = int64(options.InteractionsCacheSize) + opts.CacheSize = options.InteractionsCacheSize opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second opts.CooldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index dba17992be..9d2895bc74 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -85,6 +85,13 @@ func (iwe *InternalWrappedEvent) HasOperatorResult() bool { return iwe.OperatorsResult != nil } +func (iwe *InternalWrappedEvent) HasResults() bool { + iwe.RLock() + defer iwe.RUnlock() + + return len(iwe.Results) > 0 +} + func (iwe *InternalWrappedEvent) SetOperatorResult(operatorResult *operators.Result) { iwe.Lock() defer iwe.Unlock() diff --git a/v2/pkg/protocols/common/interactsh/const.go b/v2/pkg/protocols/common/interactsh/const.go new file mode 100644 index 0000000000..d574a121a5 --- /dev/null +++ b/v2/pkg/protocols/common/interactsh/const.go @@ -0,0 +1,18 @@ +package interactsh + +import ( + "regexp" + "time" +) + +var ( + defaultInteractionDuration = 60 * time.Second + interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`) +) + +const ( + stopAtFirstMatchAttribute = "stop-at-first-match" + templateIdAttribute = "template-id" + + defaultMaxInteractionsCount = 5000 +) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index d20d5c29ec..94e559f932 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -2,8 +2,6 @@ package interactsh import ( "bytes" - "crypto/sha1" - "encoding/hex" "fmt" "os" "regexp" @@ -12,111 +10,56 @@ import ( "sync/atomic" "time" - "github.com/karlseguin/ccache" - "github.com/pkg/errors" + "errors" + + "github.com/Mzack9999/gcache" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/interactsh/pkg/client" "github.com/projectdiscovery/interactsh/pkg/server" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/projectdiscovery/nuclei/v2/pkg/utils/atomcache" - "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" ) // Client is a wrapped client for interactsh server. type Client struct { + sync.Once + sync.RWMutex + + options *Options + // interactsh is a client for interactsh server. interactsh *client.Client // requests is a stored cache for interactsh-url->request-event data. - requests *atomcache.Cache + requests gcache.Cache[string, *RequestData] // interactions is a stored cache for interactsh-interaction->interactsh-url data - interactions *atomcache.Cache + interactions gcache.Cache[string, []*server.Interaction] // matchedTemplates is a stored cache to track matched templates - matchedTemplates *atomcache.Cache - // interactshURLs is a stored cache to track track multiple interactsh markers - interactshURLs *atomcache.Cache + matchedTemplates gcache.Cache[string, bool] + // interactshURLs is a stored cache to track multiple interactsh markers + interactshURLs gcache.Cache[string, string] - options *Options eviction time.Duration pollDuration time.Duration cooldownDuration time.Duration - dataMutex *sync.RWMutex - hostname string - firstTimeGroup sync.Once - generated uint32 // decide to wait if we have a generated url - matched atomic.Bool -} - -var ( - defaultInteractionDuration = 60 * time.Second - interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`) -) - -const ( - stopAtFirstMatchAttribute = "stop-at-first-match" - templateIdAttribute = "template-id" -) - -// Options contains configuration options for interactsh nuclei integration. -type Options struct { - // ServerURL is the URL of the interactsh server. - ServerURL string - // Authorization is the Authorization header value - Authorization string - // CacheSize is the numbers of requests to keep track of at a time. - // Older items are discarded in LRU manner in favor of new requests. - CacheSize int64 - // Eviction is the period of time after which to automatically discard - // interaction requests. - Eviction time.Duration - // CooldownPeriod is additional time to wait for interactions after closing - // of the poller. - CooldownPeriod time.Duration - // PollDuration is the time to wait before each poll to the server for interactions. - PollDuration time.Duration - // Output is the output writer for nuclei - Output output.Writer - // IssuesClient is a client for issue exporting - IssuesClient reporting.Client - // Progress is the nuclei progress bar implementation. - Progress progress.Progress - // Debug specifies whether debugging output should be shown for interactsh-client - Debug bool - DebugRequest bool - DebugResponse bool - // DisableHttpFallback controls http retry in case of https failure for server url - DisableHttpFallback bool - // NoInteractsh disables the engine - NoInteractsh bool - // NoColor dissbles printing colors for matches - NoColor bool - - StopAtFirstMatch bool - HTTPClient *retryablehttp.Client + // determines if wait the cooldown period in case of generated URL + generated atomic.Bool + matched atomic.Bool } -const defaultMaxInteractionsCount = 5000 - // New returns a new interactsh server client func New(options *Options) (*Client, error) { - configure := ccache.Configure() - configure = configure.MaxSize(options.CacheSize) - cache := atomcache.NewWithCache(ccache.New(configure)) - - interactionsCfg := ccache.Configure() - interactionsCfg = interactionsCfg.MaxSize(defaultMaxInteractionsCount) - interactionsCache := atomcache.NewWithCache(ccache.New(interactionsCfg)) - - matchedTemplateCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))) - interactshURLCache := atomcache.NewWithCache(ccache.New(ccache.Configure().MaxSize(defaultMaxInteractionsCount))) + requestsCache := gcache.New[string, *RequestData](options.CacheSize).LRU().Build() + interactionsCache := gcache.New[string, []*server.Interaction](defaultMaxInteractionsCount).LRU().Build() + matchedTemplateCache := gcache.New[string, bool](defaultMaxInteractionsCount).LRU().Build() + interactshURLCache := gcache.New[string, string](defaultMaxInteractionsCount).LRU().Build() interactClient := &Client{ eviction: options.Eviction, @@ -124,31 +67,14 @@ func New(options *Options) (*Client, error) { matchedTemplates: matchedTemplateCache, interactshURLs: interactshURLCache, options: options, - requests: cache, + requests: requestsCache, pollDuration: options.PollDuration, cooldownDuration: options.CooldownPeriod, - dataMutex: &sync.RWMutex{}, } return interactClient, nil } -// NewDefaultOptions returns the default options for interactsh client -func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options { - return &Options{ - ServerURL: client.DefaultOptions.ServerURL, - CacheSize: 5000, - Eviction: 60 * time.Second, - CooldownPeriod: 5 * time.Second, - PollDuration: 5 * time.Second, - Output: output, - IssuesClient: reporting, - Progress: progress, - DisableHttpFallback: true, - NoColor: false, - } -} - -func (c *Client) firstTimeInitializeClient() error { +func (c *Client) poll() error { if c.options.NoInteractsh { return nil // do not init if disabled } @@ -159,40 +85,34 @@ func (c *Client) firstTimeInitializeClient() error { HTTPClient: c.options.HTTPClient, }) if err != nil { - return errors.Wrap(err, "could not create client") + return errorutil.NewWithErr(err).Msgf("could not create client") } + c.interactsh = interactsh interactURL := interactsh.URL() interactDomain := interactURL[strings.Index(interactURL, ".")+1:] gologger.Info().Msgf("Using Interactsh Server: %s", interactDomain) - c.dataMutex.Lock() - c.hostname = interactDomain - c.dataMutex.Unlock() + c.setHostname(interactDomain) err = interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) { - item := c.requests.Get(interaction.UniqueID) - if item == nil { + request, err := c.requests.Get(interaction.UniqueID) + if errors.Is(err, gcache.KeyNotFoundError) || request == nil { // If we don't have any request for this ID, add it to temporary // lru cache, so we can correlate when we get an add request. - gotItem := c.interactions.Get(interaction.UniqueID) - if gotItem == nil { - c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) - } else if items, ok := gotItem.Value().([]*server.Interaction); ok { + items, err := c.interactions.Get(interaction.UniqueID) + if errorutil.IsAny(err, gcache.KeyNotFoundError) || items == nil { + _ = c.interactions.SetWithExpire(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) + } else { items = append(items, interaction) - c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration) + _ = c.interactions.SetWithExpire(interaction.UniqueID, items, defaultInteractionDuration) } return } - request, ok := item.Value().(*RequestData) - if !ok { - return - } - if _, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { - gotItem := c.matchedTemplates.Get(hash(request.Event.InternalEvent[templateIdAttribute].(string), request.Event.InternalEvent["host"].(string))) - if gotItem != nil { + if requestShouldStopAtFirstMatch(request) || c.options.StopAtFirstMatch { + if gotItem, err := c.matchedTemplates.Get(hash(request.Event.InternalEvent)); gotItem && err == nil { return } } @@ -201,23 +121,44 @@ func (c *Client) firstTimeInitializeClient() error { }) if err != nil { - return errors.Wrap(err, "could not perform instactsh polling") + return errorutil.NewWithErr(err).Msgf("could not perform interactsh polling") } return nil } +// requestShouldStopAtFirstmatch checks if furthur interactions should be stopped +// note: extra care should be taken while using this function since internalEvent is +// synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that +// we could use `TryLock()` but that may over complicate things and need to differentiate +// situations whether to block or skip +func requestShouldStopAtFirstMatch(request *RequestData) bool { + request.Event.RLock() + defer request.Event.RUnlock() + + if stop, ok := request.Event.InternalEvent[stopAtFirstMatchAttribute]; ok { + if v, ok := stop.(bool); ok { + return v + } + } + return false +} + // processInteractionForRequest processes an interaction for a request func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool { + data.Event.Lock() data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress + data.Event.Unlock() result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, c.options.Debug || c.options.DebugRequest || c.options.DebugResponse) + + // if we don't match, return if !matched || result == nil { - return false // if we don't match, return + return false } - c.requests.Delete(interaction.UniqueID) + c.requests.Remove(interaction.UniqueID) if data.Event.OperatorsResult != nil { data.Event.OperatorsResult.Merge(result) @@ -225,10 +166,12 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d data.Event.SetOperatorResult(result) } + data.Event.Lock() data.Event.Results = data.MakeResultFunc(data.Event) for _, event := range data.Event.Results { event.Interaction = interaction } + data.Event.Unlock() if c.options.Debug || c.options.DebugRequest || c.options.DebugResponse { c.debugPrintInteraction(interaction, data.Event.OperatorsResult) @@ -236,30 +179,39 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { c.matched.Store(true) - if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { - c.matchedTemplates.Set(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string)), true, defaultInteractionDuration) + if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch { + _ = c.matchedTemplates.SetWithExpire(hash(data.Event.InternalEvent), true, defaultInteractionDuration) } } + return true } +func (c *Client) AlreadyMatched(data *RequestData) bool { + data.Event.RLock() + defer data.Event.RUnlock() + + return c.matchedTemplates.Has(hash(data.Event.InternalEvent)) +} + // URL returns a new URL that can be interacted with func (c *Client) URL() (string, error) { - c.firstTimeGroup.Do(func() { - if err := c.firstTimeInitializeClient(); err != nil { - gologger.Error().Msgf("Could not initialize interactsh client: %s", err) - } - }) if c.interactsh == nil { - return "", errors.New("interactsh client not initialized") + var err error + c.Do(func() { + err = c.poll() + }) + if err != nil { + return "", errorutil.NewWithErr(err).Msgf("interactsh client not initialized") + } } - atomic.CompareAndSwapUint32(&c.generated, 0, 1) + c.generated.Store(true) return c.interactsh.URL(), nil } -// Close closes the interactsh clients after waiting for cooldown period. +// Close the interactsh clients after waiting for cooldown period. func (c *Client) Close() bool { - if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 { + if c.cooldownDuration > 0 && c.generated.Load() { time.Sleep(c.cooldownDuration) } if c.interactsh != nil { @@ -267,15 +219,10 @@ func (c *Client) Close() bool { c.interactsh.Close() } - closeCache := func(cc *atomcache.Cache) { - if cc != nil { - cc.Stop() - } - } - closeCache(c.requests) - closeCache(c.interactions) - closeCache(c.matchedTemplates) - closeCache(c.interactshURLs) + c.requests.Purge() + c.interactions.Purge() + c.matchedTemplates.Purge() + c.interactshURLs.Purge() return c.matched.Load() } @@ -308,36 +255,29 @@ func (c *Client) NewURLWithData(data string) (string, error) { if url == "" { return "", errors.New("empty interactsh url") } - c.interactshURLs.Set(url, data, defaultInteractionDuration) + _ = c.interactshURLs.SetWithExpire(url, data, defaultInteractionDuration) return url, nil } // MakePlaceholders does placeholders for interact URLs and other data to a map func (c *Client) MakePlaceholders(urls []string, data map[string]interface{}) { - data["interactsh-server"] = c.getInteractServerHostname() + data["interactsh-server"] = c.getHostname() for _, url := range urls { - if interactshURLMarker := c.interactshURLs.Get(url); interactshURLMarker != nil { - if interactshURLMarker, ok := interactshURLMarker.Value().(string); ok { - interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}") + if interactshURLMarker, err := c.interactshURLs.Get(url); interactshURLMarker != "" && err == nil { + interactshMarker := strings.TrimSuffix(strings.TrimPrefix(interactshURLMarker, "{{"), "}}") - c.interactshURLs.Delete(url) + c.interactshURLs.Remove(url) - data[interactshMarker] = url - urlIndex := strings.Index(url, ".") - if urlIndex == -1 { - continue - } - data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex] + data[interactshMarker] = url + urlIndex := strings.Index(url, ".") + if urlIndex == -1 { + continue } + data[strings.Replace(interactshMarker, "url", "id", 1)] = url[:urlIndex] } } } -// SetStopAtFirstMatch sets StopAtFirstMatch true for interactsh client options -func (c *Client) SetStopAtFirstMatch() { - c.options.StopAtFirstMatch = true -} - // MakeResultEventFunc is a result making function for nuclei type MakeResultEventFunc func(wrapped *output.InternalWrappedEvent) []*output.ResultEvent @@ -352,35 +292,26 @@ type RequestData struct { // RequestEvent is the event for a network request sent by nuclei. func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { - data.Event.Lock() - defer data.Event.Unlock() - for _, interactshURL := range interactshURLs { - id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.hostname), ".") + id := strings.TrimRight(strings.TrimSuffix(interactshURL, c.getHostname()), ".") - if _, ok := data.Event.InternalEvent[stopAtFirstMatchAttribute]; ok || c.options.StopAtFirstMatch { - gotItem := c.matchedTemplates.Get(hash(data.Event.InternalEvent[templateIdAttribute].(string), data.Event.InternalEvent["host"].(string))) - if gotItem != nil { + if requestShouldStopAtFirstMatch(data) || c.options.StopAtFirstMatch { + gotItem, err := c.matchedTemplates.Get(hash(data.Event.InternalEvent)) + if gotItem && err == nil { break } } - interaction := c.interactions.Get(id) - if interaction != nil { - // If we have previous interactions, get them and process them. - interactions, ok := interaction.Value().([]*server.Interaction) - if !ok { - c.requests.Set(id, data, c.eviction) - return - } + interactions, err := c.interactions.Get(id) + if interactions != nil && err == nil { for _, interaction := range interactions { if c.processInteractionForRequest(interaction, data) { - c.interactions.Delete(id) + c.interactions.Remove(id) break } } } else { - c.requests.Set(id, data, c.eviction) + _ = c.requests.SetWithExpire(id, data, c.eviction) } } } @@ -397,16 +328,16 @@ func HasMatchers(op *operators.Operators) bool { for _, matcher := range op.Matchers { for _, dsl := range matcher.DSL { - if strings.Contains(dsl, "interactsh") { + if stringsutil.ContainsAnyI(dsl, "interactsh") { return true } } - if strings.HasPrefix(matcher.Part, "interactsh") { + if stringsutil.HasPrefixI(matcher.Part, "interactsh") { return true } } for _, matcher := range op.Extractors { - if strings.HasPrefix(matcher.Part, "interactsh") { + if stringsutil.HasPrefixI(matcher.Part, "interactsh") { return true } } @@ -461,16 +392,22 @@ func formatInteractionMessage(key, value string, event *operators.Result, noColo return fmt.Sprintf("\n------------\n%s\n------------\n\n%s\n\n", key, value) } -func hash(templateID, host string) string { - h := sha1.New() - h.Write([]byte(templateID)) - h.Write([]byte(host)) - return hex.EncodeToString(h.Sum(nil)) +func hash(internalEvent output.InternalEvent) string { + templateId := internalEvent[templateIdAttribute].(string) + host := internalEvent["host"].(string) + return fmt.Sprintf("%s:%s", templateId, host) } -func (c *Client) getInteractServerHostname() string { - c.dataMutex.RLock() - defer c.dataMutex.RUnlock() +func (c *Client) getHostname() string { + c.RLock() + defer c.RUnlock() return c.hostname } + +func (c *Client) setHostname(hostname string) { + c.Lock() + defer c.Unlock() + + c.hostname = hostname +} diff --git a/v2/pkg/protocols/common/interactsh/options.go b/v2/pkg/protocols/common/interactsh/options.go new file mode 100644 index 0000000000..22a745d376 --- /dev/null +++ b/v2/pkg/protocols/common/interactsh/options.go @@ -0,0 +1,67 @@ +package interactsh + +import ( + "time" + + "github.com/projectdiscovery/interactsh/pkg/client" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/retryablehttp-go" +) + +// Options contains configuration options for interactsh nuclei integration. +type Options struct { + // ServerURL is the URL of the interactsh server. + ServerURL string + // Authorization is the Authorization header value + Authorization string + // CacheSize is the numbers of requests to keep track of at a time. + // Older items are discarded in LRU manner in favor of new requests. + CacheSize int + // Eviction is the period of time after which to automatically discard + // interaction requests. + Eviction time.Duration + // CooldownPeriod is additional time to wait for interactions after closing + // of the poller. + CooldownPeriod time.Duration + // PollDuration is the time to wait before each poll to the server for interactions. + PollDuration time.Duration + // Output is the output writer for nuclei + Output output.Writer + // IssuesClient is a client for issue exporting + IssuesClient reporting.Client + // Progress is the nuclei progress bar implementation. + Progress progress.Progress + // Debug specifies whether debugging output should be shown for interactsh-client + Debug bool + // DebugRequest outputs interaction request + DebugRequest bool + // DebugResponse outputs interaction response + DebugResponse bool + // DisableHttpFallback controls http retry in case of https failure for server url + DisableHttpFallback bool + // NoInteractsh disables the engine + NoInteractsh bool + // NoColor dissbles printing colors for matches + NoColor bool + + StopAtFirstMatch bool + HTTPClient *retryablehttp.Client +} + +// DefaultOptions returns the default options for interactsh client +func DefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options { + return &Options{ + ServerURL: client.DefaultOptions.ServerURL, + CacheSize: 5000, + Eviction: 60 * time.Second, + CooldownPeriod: 5 * time.Second, + PollDuration: 5 * time.Second, + Output: output, + IssuesClient: reporting, + Progress: progress, + DisableHttpFallback: true, + NoColor: false, + } +} diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index ed5f92b602..6d9a17aefe 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -186,7 +186,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { generator.options.Interactsh, err = interactsh.New(&interactsh.Options{ ServerURL: options.InteractshURL, - CacheSize: int64(options.InteractionsCacheSize), + CacheSize: options.InteractionsCacheSize, Eviction: time.Duration(options.InteractionsEviction) * time.Second, CooldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0e80eca0c2..4443ba9873 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -248,24 +248,26 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous } var gotMatches bool requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) { - // Add the extracts to the dynamic values if any. - if event.OperatorsResult != nil { - gotMatches = event.OperatorsResult.Matched - } if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(gr.InteractURLs, &interactsh.RequestData{ + requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, MatchFunc: request.Match, ExtractFunc: request.Extract, - }) + } + request.options.Interactsh.RequestEvent(gr.InteractURLs, requestData) + gotMatches = request.options.Interactsh.AlreadyMatched(requestData) } else { callback(event) } + // Add the extracts to the dynamic values if any. + if event.OperatorsResult != nil { + gotMatches = event.OperatorsResult.Matched + } }, 0) // If a variable is unresolved, skip all further requests - if requestErr == errStopExecution { + if errors.Is(requestErr, errStopExecution) { return false } if requestErr != nil { @@ -276,7 +278,8 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous request.options.Progress.IncrementRequests() // If this was a match, and we want to stop at first match, skip all further requests. - if (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches { + shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch + if shouldStopAtFirstMatch && gotMatches { return false } return true @@ -374,19 +377,21 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } var gotMatches bool err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { - // Add the extracts to the dynamic values if any. - if event.OperatorsResult != nil { - gotMatches = event.OperatorsResult.Matched - gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues) - } if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{ + requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, MatchFunc: request.Match, ExtractFunc: request.Extract, - }) + } + request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, requestData) + gotMatches = request.options.Interactsh.AlreadyMatched(requestData) + } + // Add the extracts to the dynamic values if any. + if event.OperatorsResult != nil { + gotMatches = event.OperatorsResult.Matched + gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues) } // Note: This is a race condition prone zone i.e when request has interactsh_matchers // Interactsh.RequestEvent tries to access/update output.InternalWrappedEvent depending on logic @@ -397,7 +402,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa }, generator.currentIndex) // If a variable is unresolved, skip all further requests - if err == errStopExecution { + if errors.Is(err, errStopExecution) { return true, nil } if err != nil { @@ -409,7 +414,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.Progress.IncrementRequests() // If this was a match, and we want to stop at first match, skip all further requests. - if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotMatches { + shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch + if shouldStopAtFirstMatch && gotMatches { return true, nil } return false, nil @@ -755,7 +761,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ callback(event) // Skip further responses if we have stop-at-first-match and a match - if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && len(event.Results) > 0 { + if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() { return nil } } diff --git a/v2/pkg/utils/atomcache/atomcache.go b/v2/pkg/utils/atomcache/atomcache.go deleted file mode 100644 index ee6cc84b7d..0000000000 --- a/v2/pkg/utils/atomcache/atomcache.go +++ /dev/null @@ -1,62 +0,0 @@ -package atomcache - -import ( - "sync" - "sync/atomic" - "time" - - "github.com/karlseguin/ccache" -) - -type Cache struct { - *ccache.Cache - Closed atomic.Bool - mu sync.RWMutex -} - -func NewWithCache(c *ccache.Cache) *Cache { - return &Cache{Cache: c} -} - -func (c *Cache) Get(key string) *ccache.Item { - if c.Closed.Load() { - return nil - } - c.mu.RLock() - defer c.mu.RUnlock() - - return c.Cache.Get(key) -} - -func (c *Cache) Set(key string, value interface{}, duration time.Duration) { - if c.Closed.Load() { - return - } - c.mu.Lock() - defer c.mu.Unlock() - - c.Cache.Set(key, value, duration) -} - -func (c *Cache) Delete(key string) bool { - if c.Closed.Load() { - return false - } - - c.mu.Lock() - defer c.mu.Unlock() - - return c.Cache.Delete(key) -} - -func (c *Cache) Stop() { - if c.Closed.Load() { - return - } - c.Closed.Store(true) - - c.mu.Lock() - defer c.mu.Unlock() - - c.Cache.Stop() -} From 786ee982ed1ffc71ebcc6d46b91f0755ba397e58 Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Mon, 17 Apr 2023 04:18:06 -0400 Subject: [PATCH 23/34] Azure Blob Storage Custom Template Provider (#3542) * Initial library creation for downloading blobs from Azure * Environment variable hooks for Azure config values * Missing configuration checks for Azure blob connection * Package dependencies for Azure client library * Make output directory if it doesn't exist, add log for downloaded templates * Add example formats for Azure properties * Add path to the output after download * Only download .yaml files instead of also .yml .json --- v2/go.mod | 9 +- v2/go.sum | 17 +++ v2/internal/runner/options.go | 38 ++++++ v2/pkg/catalog/config/config.go | 1 + v2/pkg/external/customtemplates/azure_blob.go | 121 ++++++++++++++++++ .../customtemplates/templates_provider.go | 20 ++- v2/pkg/types/types.go | 10 ++ 7 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 v2/pkg/external/customtemplates/azure_blob.go diff --git a/v2/go.mod b/v2/go.mod index 4c0d3f6ea6..ea3d70ec5e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -51,6 +51,8 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/DataDog/gostackparse v0.6.0 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 @@ -89,6 +91,9 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect @@ -112,12 +117,14 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.8 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect github.com/minio/selfupdate v0.6.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/projectdiscovery/asnmap v1.0.2 // indirect github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect github.com/projectdiscovery/freeport v0.0.4 // indirect @@ -159,7 +166,7 @@ require ( github.com/goburrow/cache v0.1.4 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/v2/go.sum b/v2/go.sum index e88a6ba176..656f1555a9 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -2,6 +2,16 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1vBI4o= github.com/DataDog/gostackparse v0.6.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= @@ -203,6 +213,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -317,6 +329,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= @@ -402,6 +416,8 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -724,6 +740,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 46d34db761..3d880e125a 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -157,6 +157,14 @@ func validateOptions(options *types.Options) error { } } + // Verify Azure connection configuration is passed if the Azure template bucket is passed + if options.AzureContainerName != "" && options.UpdateTemplates { + missing := validateMissingAzureOptions(options) + if missing != nil { + return fmt.Errorf("azure connection details are missing. Please provide %s", strings.Join(missing, ",")) + } + } + // verify that a valid ip version type was selected (4, 6) if len(options.IPVersion) == 0 { // add ipv4 as default @@ -198,6 +206,8 @@ func validateCloudOptions(options *types.Options) error { missing = validateMissingS3Options(options) case "github": missing = validateMissingGithubOptions(options) + case "azure": + missing = validateMissingAzureOptions(options) } if len(missing) > 0 { return fmt.Errorf("missing %v env variables", strings.Join(missing, ", ")) @@ -223,6 +233,26 @@ func validateMissingS3Options(options *types.Options) []string { return missing } +func validateMissingAzureOptions(options *types.Options) []string { + var missing []string + if options.AzureTenantID == "" { + missing = append(missing, "AZURE_TENANT_ID") + } + if options.AzureClientID == "" { + missing = append(missing, "AZURE_CLIENT_ID") + } + if options.AzureClientSecret == "" { + missing = append(missing, "AZURE_CLIENT_SECRET") + } + if options.AzureServiceURL == "" { + missing = append(missing, "AZURE_SERVICE_URL") + } + if options.AzureContainerName == "" { + missing = append(missing, "AZURE_CONTAINER_NAME") + } + return missing +} + func validateMissingGithubOptions(options *types.Options) []string { var missing []string if options.GithubToken == "" { @@ -323,8 +353,16 @@ func readEnvInputVars(options *types.Options) { if repolist != "" { options.GithubTemplateRepo = append(options.GithubTemplateRepo, stringsutil.SplitAny(repolist, ",")...) } + // AWS options for downloading templates from an S3 bucket options.AwsAccessKey = os.Getenv("AWS_ACCESS_KEY") options.AwsSecretKey = os.Getenv("AWS_SECRET_KEY") options.AwsBucketName = os.Getenv("AWS_TEMPLATE_BUCKET") options.AwsRegion = os.Getenv("AWS_REGION") + + // Azure options for downloading templates from an Azure Blob Storage container + options.AzureContainerName = os.Getenv("AZURE_CONTAINER_NAME") + options.AzureTenantID = os.Getenv("AZURE_TENANT_ID") + options.AzureClientID = os.Getenv("AZURE_CLIENT_ID") + options.AzureClientSecret = os.Getenv("AZURE_CLIENT_SECRET") + options.AzureServiceURL = os.Getenv("AZURE_SERVICE_URL") } diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index ad0b04cc7e..44e3141acb 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -19,6 +19,7 @@ type Config struct { CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` + CustomAzureTemplatesDirectory string `json:"custom-azure-templates-directory"` TemplateVersion string `json:"nuclei-templates-version,omitempty"` NucleiVersion string `json:"nuclei-version,omitempty"` diff --git a/v2/pkg/external/customtemplates/azure_blob.go b/v2/pkg/external/customtemplates/azure_blob.go new file mode 100644 index 0000000000..76a6ea9144 --- /dev/null +++ b/v2/pkg/external/customtemplates/azure_blob.go @@ -0,0 +1,121 @@ +package customtemplates + +import ( + "bytes" + "context" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/projectdiscovery/gologger" + "os" + "path/filepath" + "strings" +) + +type customTemplateAzureBlob struct { + azureBlobClient *azblob.Client + containerName string +} + +func getAzureBlobClient(tenantID string, clientID string, clientSecret string, serviceURL string) (*azblob.Client, error) { + // Create an Azure credential using the provided credentials + credentials, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) + if err != nil { + gologger.Error().Msgf("Invalid Azure credentials: %v", err) + return nil, err + } + + // Create a client to manage Azure Blob Storage + client, err := azblob.NewClient(serviceURL, credentials, nil) + if err != nil { + gologger.Error().Msgf("Error creating Azure Blob client: %v", err) + return nil, err + } + + return client, nil +} + +func (bk *customTemplateAzureBlob) Download(location string, ctx context.Context) { + // Set an incrementer for the number of templates downloaded + var templatesDownloaded = 0 + + // Define the local path to which the templates will be downloaded + downloadPath := filepath.Join(location, CustomAzureTemplateDirectory, bk.containerName) + + // Get the list of all templates from the container + pager := bk.azureBlobClient.NewListBlobsFlatPager(bk.containerName, &azblob.ListBlobsFlatOptions{ + // Don't include previous versions of the templates if versioning is enabled on the container + Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false}, + }) + + // Loop through the list of blobs in the container and determine if they should be added to the list of templates + // to be returned, and subsequently downloaded + for pager.More() { + resp, err := pager.NextPage(context.TODO()) + if err != nil { + gologger.Error().Msgf("Error listing templates in Azure Blob container: %v", err) + return + } + + for _, blob := range resp.Segment.BlobItems { + // If the blob is a .yaml download the file to the local filesystem + if strings.HasSuffix(*blob.Name, ".yaml") { + // Download the template to the local filesystem at the downloadPath + err := downloadTemplate(bk.azureBlobClient, bk.containerName, *blob.Name, filepath.Join(downloadPath, *blob.Name), ctx) + if err != nil { + gologger.Error().Msgf("Error downloading template: %v", err) + } else { + // Increment the number of templates downloaded + templatesDownloaded++ + } + } + } + } + + // Log the number of templates downloaded + gologger.Info().Msgf("Downloaded %d templates from Azure Blob Storage container '%s' to: %s", templatesDownloaded, bk.containerName, downloadPath) +} + +// Update updates the templates from the Azure Blob Storage container to the local filesystem. This is effectively a +// wrapper of the Download function which downloads of all templates from the container and doesn't manage a +// differential update. +func (bk *customTemplateAzureBlob) Update(location string, ctx context.Context) { + // Treat the update as a download of all templates from the container + bk.Download(location, ctx) +} + +// downloadTemplate downloads a template from the Azure Blob Storage container to the local filesystem with the provided +// blob path and outputPath. +func downloadTemplate(client *azblob.Client, containerName string, path string, outputPath string, ctx context.Context) error { + // Download the blob as a byte stream + get, err := client.DownloadStream(ctx, containerName, path, nil) + if err != nil { + gologger.Error().Msgf("Error downloading template: %v", err) + return err + } + + downloadedData := bytes.Buffer{} + retryReader := get.NewRetryReader(ctx, &azblob.RetryReaderOptions{}) + _, err = downloadedData.ReadFrom(retryReader) + if err != nil { + gologger.Error().Msgf("Error reading template: %v", err) + return err + } + + err = retryReader.Close() + if err != nil { + gologger.Error().Msgf("Error closing template filestream: %v", err) + return err + } + + // Ensure the directory exists + err = os.MkdirAll(filepath.Dir(outputPath), 0755) + if err != nil { + gologger.Error().Msgf("Error creating directory: %v", err) + return err + } + + // Write the downloaded template to the local filesystem at the outputPath with the filename of the blob name + err = os.WriteFile(outputPath, downloadedData.Bytes(), 0644) + + return err +} diff --git a/v2/pkg/external/customtemplates/templates_provider.go b/v2/pkg/external/customtemplates/templates_provider.go index e4b5f0ed55..ebf790c74d 100644 --- a/v2/pkg/external/customtemplates/templates_provider.go +++ b/v2/pkg/external/customtemplates/templates_provider.go @@ -11,6 +11,7 @@ import ( const ( CustomGithubTemplateDirectory = "github" CustomS3TemplateDirectory = "s3" + CustomAzureTemplateDirectory = "azure" ) type Provider interface { @@ -18,7 +19,7 @@ type Provider interface { Update(location string, ctx context.Context) } -// parseCustomTemplates function reads the options.GithubTemplateRepo list, +// ParseCustomTemplates function reads the options.GithubTemplateRepo list, // Checks the given repos are valid or not and stores them into runner.CustomTemplates func ParseCustomTemplates(options *types.Options) []Provider { if options.Cloud { @@ -63,5 +64,22 @@ func ParseCustomTemplates(options *types.Options) []Provider { } customTemplates = append(customTemplates, ctBucket) } + if options.AzureContainerName != "" { + // Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage + azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL) + if err != nil { + gologger.Error().Msgf("Error establishing Azure Blob client for %s %s", options.AzureContainerName, err) + return customTemplates + } + + // Create a new Azure Blob Storage container object + azTemplateContainer := &customTemplateAzureBlob{ + azureBlobClient: azClient, + containerName: options.AzureContainerName, + } + + // Add the Azure Blob Storage container object to the list of custom templates + customTemplates = append(customTemplates, azTemplateContainer) + } return customTemplates } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 838981e3b3..10ea6d8972 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -348,6 +348,16 @@ type Options struct { AwsBucketName string // AWS Region name where aws s3 bucket is located AwsRegion string + // AzureContainerName for downloading templates from Azure Blob Storage. Example: templates + AzureContainerName string + // AzureTenantID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000 + AzureTenantID string + // AzureClientID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000 + AzureClientID string + // AzureClientSecret for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000 + AzureClientSecret string + // AzureServiceURL for downloading templates from Azure Blob Storage. Example: https://XXXXXXXXXX.blob.core.windows.net/ + AzureServiceURL string // Scan Strategy (auto,hosts-spray,templates-spray) ScanStrategy string // Fuzzing Type overrides template level fuzzing-type configuration From fdc4b45f3f113ac16b70a8cb066f3fa8a9b0a0c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 13:48:49 +0530 Subject: [PATCH 24/34] chore(deps): bump github.com/rs/xid from 1.4.0 to 1.5.0 in /v2 (#3556) Bumps [github.com/rs/xid](https://github.com/rs/xid) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/rs/xid/releases) - [Commits](https://github.com/rs/xid/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: github.com/rs/xid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index ea3d70ec5e..c26a6e157c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/rs/xid v1.4.0 + github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 github.com/shirou/gopsutil/v3 v3.22.12 github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 656f1555a9..f22002a10e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -492,8 +492,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= From 93639f588274395badc56e0dd87399ac8c7038dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 13:49:22 +0530 Subject: [PATCH 25/34] chore(deps): bump github.com/xanzy/go-gitlab in /v2 (#3555) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.81.0 to 0.82.0. - [Release notes](https://github.com/xanzy/go-gitlab/releases) - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.81.0...v0.82.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index c26a6e157c..5574d78504 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -41,7 +41,7 @@ require ( github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.30.0 - github.com/xanzy/go-gitlab v0.81.0 + github.com/xanzy/go-gitlab v0.82.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.8.0 golang.org/x/oauth2 v0.6.0 diff --git a/v2/go.sum b/v2/go.sum index f22002a10e..e04cdf21b0 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -596,8 +596,8 @@ github.com/weppos/publicsuffix-go v0.30.0 h1:QHPZ2GRu/YE7cvejH9iyavPOkVCB4dNxp2Z github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= -github.com/xanzy/go-gitlab v0.81.0 h1:ofbhZ5ZY9AjHATWQie4qd2JfncdUmvcSA/zfQB767Dk= -github.com/xanzy/go-gitlab v0.81.0/go.mod h1:VMbY3JIWdZ/ckvHbQqkyd3iYk2aViKrNIQ23IbFMQDo= +github.com/xanzy/go-gitlab v0.82.0 h1:WUAqZqNj/VGsXvFM9mYC0MEpPpmK4sHoOAryRQJARp4= +github.com/xanzy/go-gitlab v0.82.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= From 97daad20350d4448a4e9decdd401a382fddecd64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 13:49:52 +0530 Subject: [PATCH 26/34] chore(deps): bump github.com/klauspost/compress in /v2 (#3557) Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.16.0 to 1.16.4. - [Release notes](https://github.com/klauspost/compress/releases) - [Changelog](https://github.com/klauspost/compress/blob/master/.goreleaser.yml) - [Commits](https://github.com/klauspost/compress/compare/v1.16.0...v1.16.4) --- updated-dependencies: - dependency-name: github.com/klauspost/compress dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5574d78504..e758b56447 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -66,7 +66,7 @@ require ( github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.6.1 github.com/h2non/filetype v1.1.3 - github.com/klauspost/compress v1.16.0 + github.com/klauspost/compress v1.16.4 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible github.com/mitchellh/go-homedir v1.1.0 diff --git a/v2/go.sum b/v2/go.sum index e04cdf21b0..9dbf6ee33e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -311,8 +311,8 @@ github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= +github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= From 3d5203617817448f259af8c19f5dd3d8a264d460 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 13:50:08 +0530 Subject: [PATCH 27/34] chore(deps): bump github.com/shirou/gopsutil/v3 in /v2 (#3558) Bumps [github.com/shirou/gopsutil/v3](https://github.com/shirou/gopsutil) from 3.22.12 to 3.23.3. - [Release notes](https://github.com/shirou/gopsutil/releases) - [Commits](https://github.com/shirou/gopsutil/compare/v3.22.12...v3.23.3) --- updated-dependencies: - dependency-name: github.com/shirou/gopsutil/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 3 ++- v2/go.sum | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e758b56447..39a05b35c6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -34,7 +34,7 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.22.12 + github.com/shirou/gopsutil/v3 v3.23.3 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.0 github.com/syndtr/goleveldb v1.0.0 @@ -128,6 +128,7 @@ require ( github.com/projectdiscovery/asnmap v1.0.2 // indirect github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect github.com/projectdiscovery/freeport v0.0.4 // indirect + github.com/shoenig/go-m1cpu v0.1.4 // indirect github.com/skeema/knownhosts v1.1.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.2.10 // indirect diff --git a/v2/go.sum b/v2/go.sum index 9dbf6ee33e..4cb5e7cf2f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -503,8 +503,12 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4 h1:SZPIgRM2sEF9NJy50mRHu9PKGwxyyTTJIWvCtgVbozs= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -758,6 +762,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From 2d769f1b4ace4aafbbddafd00fee1d9c90a9ff05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 13:50:22 +0530 Subject: [PATCH 28/34] chore(deps): bump github.com/aws/aws-sdk-go-v2/credentials in /v2 (#3559) Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.13.19 to 1.13.20. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.13.19...credentials/v1.13.20) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 8 ++++---- v2/go.sum | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 39a05b35c6..1650db8456 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -59,7 +59,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go-v2 v1.17.8 github.com/aws/aws-sdk-go-v2/config v1.18.20 - github.com/aws/aws-sdk-go-v2/credentials v1.13.19 + github.com/aws/aws-sdk-go-v2/credentials v1.13.20 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61 github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2 github.com/docker/go-units v0.5.0 @@ -237,9 +237,9 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 4cb5e7cf2f..bd1db6f3b3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -80,8 +80,9 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.20 h1:yYy+onqmLmDVZtx0mkqbx8aJPl+58V6ivLbLDZ2Qztc= github.com/aws/aws-sdk-go-v2/config v1.18.20/go.mod h1:RWjF39RiDevmHw/+VaD8F0A36OPIPTHQQyRx0eZohnw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.19 h1:FWHJy9uggyQCSEhovtl/6W6rW9P6DSr62GUeY/TS6Eo= github.com/aws/aws-sdk-go-v2/credentials v1.13.19/go.mod h1:2m4uvLvl5hvQezVkLeBBUGMEDm5GcUNc3016W6d3NGg= +github.com/aws/aws-sdk-go-v2/credentials v1.13.20 h1:oZCEFcrMppP/CNiS8myzv9JgOzq2s0d3v3MXYil/mxQ= +github.com/aws/aws-sdk-go-v2/credentials v1.13.20/go.mod h1:xtZnXErtbZ8YGXC3+8WfajpMBn5Ga/3ojZdxHq6iI8o= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 h1:jOzQAesnBFDmz93feqKnsTHsXrlwWORNZMFHMV+WLFU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2/go.mod h1:cDh1p6XkSGSwSRIArWRc6+UqAQ7x4alQ0QfpVR6f+co= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.61 h1:0fHTNkoMAz7jbXSyo0SLubbTJEO+AgvkZ0iWT9DbEsk= @@ -104,12 +105,15 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1 h1:lRWp3bNu5wy0X3 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.1/go.mod h1:VXBHSxdN46bsJrkniN68psSwbyBKsazQfU2yX/iSDso= github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2 h1:iOZoYePk+EuBI1tC7bxeRjO+JvClcYm2fZYW5WPIOMQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.31.2/go.mod h1:aSl9/LJltSz1cVusiR/Mu8tvI4Sv/5w/WWrJmmkNii0= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.7 h1:rrYYhsvcvg6CDDoo4GHKtAWBFutS86CpmGvqHJHYL9w= github.com/aws/aws-sdk-go-v2/service/sso v1.12.7/go.mod h1:GNIveDnP+aE3jujyUSH5aZ/rktsTM5EvtKnCqBZawdw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7 h1:Vjpjt3svuJ/u+eKRfycZwqLsLoxyuvvZyHMJSk+3k58= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.8 h1:5cb3D6xb006bPTqEfCNaEA6PPEfBXxxy4NNeX/44kGk= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.8/go.mod h1:GNIveDnP+aE3jujyUSH5aZ/rktsTM5EvtKnCqBZawdw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.7/go.mod h1:44qFP1g7pfd+U+sQHLPalAPKnyfTZjJsYR4xIwsJy5o= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.8 h1:SQ8pPoXfzuz4DImO4KAi9xhO4ANG0Ckb5clZ5GoRAPw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8 h1:NZaj0ngZMzsubWZbrEFSB4rgSQRbFq38Sd6KBxHuOIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8/go.mod h1:44qFP1g7pfd+U+sQHLPalAPKnyfTZjJsYR4xIwsJy5o= github.com/aws/aws-sdk-go-v2/service/sts v1.18.8/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 h1:Qf1aWwnsNkyAoqDqmdM3nHwN78XQjec27LjM6b9vyfI= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.9/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= From d0f22f8b7374686aea4ecfe1609be73393afe22e Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Wed, 19 Apr 2023 01:57:53 +0530 Subject: [PATCH 29/34] Issue 3488 http race (#3533) * debug * Add body after the request creation * fix race_count template hangs * remove printf * update if condition --- v2/go.mod | 4 ++-- v2/go.sum | 12 ++++++------ v2/pkg/protocols/http/build_request.go | 9 +++++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 1650db8456..2789afc8b3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -43,8 +43,8 @@ require ( github.com/weppos/publicsuffix-go v0.30.0 github.com/xanzy/go-gitlab v0.82.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.6.0 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.7.0 golang.org/x/text v0.9.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 diff --git a/v2/go.sum b/v2/go.sum index bd1db6f3b3..d22f6270a7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -161,6 +161,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= @@ -215,7 +216,6 @@ github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -712,11 +712,11 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -777,7 +777,7 @@ golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ae9b9e3f26..d1e0642837 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -268,14 +268,14 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs} return unsafeReq, nil } - var body io.ReadCloser body = io.NopCloser(strings.NewReader(rawRequestData.Data)) - if r.request.Race { + if r.request.Race && r.request.RaceNumberRequests > 0 { // More or less this ensures that all requests hit the endpoint at the same approximated time // Todo: sync internally upon writing latest request byte body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second) } + urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw") @@ -284,6 +284,11 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st if err != nil { return nil, err } + // override the body with a new one that will be used to read the request body in parallel threads + // for race condition testing + if r.request.Threads > 0 && r.request.Race { + req.Body = race.NewOpenGateWithTimeout(req.Body, time.Duration(2)*time.Second) + } for key, value := range rawRequestData.Headers { if key == "" { continue From bf08913cd056dd946378d85828741106e146b2c4 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Wed, 19 Apr 2023 21:58:48 +0530 Subject: [PATCH 30/34] update logic + config management refactor (#3567) * adds template manager * refactor: checkpoint * centrailized config & template download logic * refactor removed unused code * use global template directory * update related bug fixes * bug fix create cfg dir if missing * fix lint error * bug fix skip writing template dir in callback * misc update * remove unused code * use strings.equalfold for comparison --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- v2/cmd/integration-test/code.go | 6 +- v2/cmd/integration-test/template-path.go | 5 +- v2/cmd/nuclei/main.go | 122 ++-- v2/examples/simple.go | 6 +- v2/go.mod | 16 +- v2/go.sum | 62 +- v2/internal/installer/doc.go | 6 + v2/internal/installer/template.go | 328 +++++++++ v2/internal/installer/template_test.go | 59 ++ v2/internal/installer/util.go | 69 ++ v2/internal/installer/versioncheck.go | 91 +++ v2/internal/installer/versioncheck_test.go | 19 + v2/internal/installer/zipslip_unix_test.go | 68 ++ v2/internal/runner/banner.go | 9 +- v2/internal/runner/healthcheck.go | 12 +- v2/internal/runner/nucleicloud/utils.go | 5 +- v2/internal/runner/options.go | 28 +- v2/internal/runner/runner.go | 209 ++---- v2/internal/runner/templates.go | 9 +- v2/internal/runner/update.go | 646 ------------------ v2/internal/runner/update_test.go | 162 ----- v2/internal/runner/update_unix_test.go | 34 - v2/pkg/catalog/config/config.go | 183 ----- v2/pkg/catalog/config/constants.go | 48 ++ v2/pkg/catalog/config/ignorefile.go | 31 + v2/pkg/catalog/config/nucleiconfig.go | 302 ++++++++ v2/pkg/catalog/config/template.go | 7 + v2/pkg/catalog/loader/loader.go | 12 +- v2/pkg/catalog/loader/loader_test.go | 15 +- v2/pkg/parsers/workflow_loader.go | 3 +- .../common/automaticscan/automaticscan.go | 2 +- .../protocols/common/generators/generators.go | 5 +- .../common/generators/generators_test.go | 6 +- v2/pkg/protocols/headless/headless.go | 2 +- v2/pkg/protocols/http/http.go | 2 +- .../protocols/http/request_generator_test.go | 4 +- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/websocket/websocket.go | 2 +- v2/pkg/reporting/reporting.go | 9 +- v2/pkg/reporting/trackers/jira/jira.go | 2 +- v2/pkg/templates/compile_test.go | 3 +- v2/pkg/testutils/testutils.go | 6 +- v2/pkg/types/resume.go | 5 +- v2/pkg/types/types.go | 14 +- v2/pkg/utils/template_path.go | 20 +- 45 files changed, 1216 insertions(+), 1440 deletions(-) create mode 100644 v2/internal/installer/doc.go create mode 100644 v2/internal/installer/template.go create mode 100644 v2/internal/installer/template_test.go create mode 100644 v2/internal/installer/util.go create mode 100644 v2/internal/installer/versioncheck.go create mode 100644 v2/internal/installer/versioncheck_test.go create mode 100644 v2/internal/installer/zipslip_unix_test.go delete mode 100644 v2/internal/runner/update.go delete mode 100644 v2/internal/runner/update_test.go delete mode 100644 v2/internal/runner/update_unix_test.go delete mode 100644 v2/pkg/catalog/config/config.go create mode 100644 v2/pkg/catalog/config/constants.go create mode 100644 v2/pkg/catalog/config/ignorefile.go create mode 100644 v2/pkg/catalog/config/nucleiconfig.go diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index eb53be5c69..408ccbf5ac 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -120,11 +120,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { } executerOpts.WorkflowLoader = workflowLoader - configObject, err := config.ReadConfiguration() - if err != nil { - return nil, errors.Wrap(err, "could not read configuration file") - } - store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts)) + store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) if err != nil { return nil, errors.Wrap(err, "could not create loader") } diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 984fe29642..9633558d8d 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -4,13 +4,12 @@ import ( "fmt" "strings" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) func getTemplatePath() string { - templatePath, _ := utils.GetDefaultTemplatePath() - return templatePath + return config.DefaultConfig.TemplatesDirectory } var templatesPathTestCases = map[string]testutils.TestCase{ diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 60f2d6b8f1..4f7c4316ad 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -1,14 +1,12 @@ package main import ( - "errors" "fmt" - "io" "os" "os/signal" - "path/filepath" "runtime" "runtime/pprof" + "strings" "time" "github.com/projectdiscovery/goflags" @@ -39,8 +37,7 @@ func main() { if err := runner.ConfigureOptions(); err != nil { gologger.Fatal().Msgf("Could not initialize options: %s\n", err) } - flagSet := readConfig() - configPath, _ := flagSet.GetConfigFilePath() + _ = readConfig() if options.ListDslSignatures { gologger.Info().Msgf("The available custom DSL functions are:") @@ -67,7 +64,6 @@ func main() { } runner.ParseOptions(options) - options.ConfigPath = configPath if options.HangMonitor { cancel := monitor.NewStackMonitor(10 * time.Second) @@ -117,7 +113,6 @@ func main() { } func readConfig() *goflags.FlagSet { - flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing on extensive configurability, massive extensibility and ease of use.`) @@ -171,7 +166,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"), flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), - flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isNoColorEnabled(), "disable output content coloring (ANSI escape codes)"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isColorNotAvailable(), "disable output content coloring (ANSI escape codes)"), flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"), @@ -284,22 +279,22 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.ListDslSignatures, "list-dsl-function", "ldf", false, "list all supported DSL function signatures"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), - flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), + flagSet.CallbackVar(printVersion, "version", "show nuclei version"), flagSet.BoolVarP(&options.HangMonitor, "hang-monitor", "hm", false, "enable nuclei hang monitoring"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.StringVar(&memProfile, "profile-mem", "", "optional nuclei memory profile dump file"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"), flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"), - flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), + flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"), flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"), ) flagSet.CreateGroup("update", "Update", - flagSet.BoolVarP(&options.UpdateNuclei, "update", "un", false, "update nuclei engine to the latest released version"), + flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "un", "update nuclei engine to the latest released version"), flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"), - flagSet.StringVarP(&options.TemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"), - flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"), + flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"), + flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"), ) flagSet.CreateGroup("stats", "Statistics", @@ -342,52 +337,34 @@ on extensive configurability, massive extensibility and ease of use.`) http.LeaveDefaultPorts = true } if options.CustomConfigDir != "" { - originalIgnorePath := config.GetIgnoreFilePath() - config.SetCustomConfigDirectory(options.CustomConfigDir) - configPath := filepath.Join(options.CustomConfigDir, "config.yaml") - ignoreFile := filepath.Join(options.CustomConfigDir, ".nuclei-ignore") - if !fileutil.FileExists(ignoreFile) { - if err := fileutil.CopyFile(originalIgnorePath, ignoreFile); err != nil { - gologger.Error().Msgf("failed to copy .nuclei-ignore file in custom config directory got %v", err) - } - } - readConfigFile := func() error { - if err := flagSet.MergeConfigFile(configPath); err != nil && !errors.Is(err, io.EOF) { - defaultConfigPath, _ := flagSet.GetConfigFilePath() - err = fileutil.CopyFile(defaultConfigPath, configPath) - if err != nil { - return err - } - return errors.New("reload the config file") - } - return nil - } - if err := readConfigFile(); err != nil { - _ = readConfigFile() - } + config.DefaultConfig.SetConfigDir(options.CustomConfigDir) + readFlagsConfig(flagSet) } if cfgFile != "" { + if !fileutil.FileExists(cfgFile) { + gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile) + } + // merge config file with flags if err := flagSet.MergeConfigFile(cfgFile); err != nil { gologger.Fatal().Msgf("Could not read config: %s\n", err) } - cfgFileFolder := filepath.Dir(cfgFile) - if err := config.OverrideIgnoreFilePath(cfgFileFolder); err != nil { - gologger.Warning().Msgf("Could not read ignore file from custom path: %s\n", err) - } } + if options.NewTemplatesDirectory != "" { + config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory) + } + cleanupOldResumeFiles() return flagSet } -func isNoColorEnabled() bool { +// isColorNotAvailable returns true if ascii collored output is not available. +func isColorNotAvailable() bool { return runtime.GOOS == "windows" } +// cleanupOldResumeFiles cleans up resume files older than 10 days. func cleanupOldResumeFiles() { - root, err := config.GetConfigDir() - if err != nil { - return - } + root := config.DefaultConfig.GetConfigDir() filter := fileutil.FileFilters{ OlderThan: 24 * time.Hour * 10, // cleanup on the 10th day Prefix: "resume-", @@ -395,9 +372,62 @@ func cleanupOldResumeFiles() { _ = fileutil.DeleteFilesOlderThan(root, filter) } +// readFlagsConfig reads the config file from the default config dir and copies it to the current config dir. +func readFlagsConfig(flagset *goflags.FlagSet) { + // check if config.yaml file exists + defaultCfgFile, err := flagset.GetConfigFilePath() + if err != nil { + // something went wrong either dir is not readable or something else went wrong upstream in `goflags` + // warn and exit in this case + gologger.Warning().Msgf("Could not read config file: %s\n", err) + return + } + cfgFile := config.DefaultConfig.GetFlagsConfigFilePath() + if !fileutil.FileExists(cfgFile) { + if !fileutil.FileExists(defaultCfgFile) { + // if default config does not exist, warn and exit + gologger.Warning().Msgf("missing default config file : %s", defaultCfgFile) + return + } + // if does not exist copy it from the default config + if err = fileutil.CopyFile(defaultCfgFile, cfgFile); err != nil { + gologger.Warning().Msgf("Could not copy config file: %s\n", err) + } + return + } + // if config file exists, merge it with the default config + if err = flagset.MergeConfigFile(cfgFile); err != nil { + gologger.Warning().Msgf("failed to merge configfile with flags got: %s\n", err) + } +} + +// disableUpdatesCallback disables the update check. +func disableUpdatesCallback() { + config.DefaultConfig.DisableUpdateCheck() +} + +// printVersion prints the nuclei version and exits. +func printVersion() { + gologger.Info().Msgf("Nuclei Engine Version: %s", config.Version) + os.Exit(0) +} + +// printTemplateVersion prints the nuclei template version and exits. +func printTemplateVersion() { + cfg := config.DefaultConfig + gologger.Info().Msgf("Public nuclei-templates version: %s (%s)\n", cfg.TemplateVersion, cfg.TemplatesDirectory) + if cfg.CustomS3TemplatesDirectory != "" { + gologger.Info().Msgf("Custom S3 templates location: %s\n", cfg.CustomS3TemplatesDirectory) + } + if cfg.CustomGithubTemplatesDirectory != "" { + gologger.Info().Msgf("Custom Github templates location: %s ", cfg.CustomGithubTemplatesDirectory) + } + os.Exit(0) +} + func init() { // print stacktrace of errors in debug mode - if os.Getenv("DEBUG") != "" { + if strings.EqualFold(os.Getenv("DEBUG"), "true") { errorutil.ShowStackTrace = true } } diff --git a/v2/examples/simple.go b/v2/examples/simple.go index 5b80d22821..1550fbbc7b 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -80,11 +80,7 @@ func main() { } executerOpts.WorkflowLoader = workflowLoader - configObject, err := config.ReadConfiguration() - if err != nil { - log.Fatalf("Could not read config: %s\n", err) - } - store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts)) + store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) if err != nil { log.Fatalf("Could not create loader client: %s\n", err) } diff --git a/v2/go.mod b/v2/go.mod index 2789afc8b3..88d839acf8 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,8 +7,6 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.16.0 github.com/antchfx/htmlquery v1.3.0 - github.com/apex/log v1.9.0 - github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/go-playground/validator/v10 v10.11.2 @@ -25,7 +23,7 @@ require ( github.com/projectdiscovery/clistats v0.0.12 github.com/projectdiscovery/fastdialer v0.0.24 github.com/projectdiscovery/hmap v0.0.11 - github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d + github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467 github.com/projectdiscovery/rawhttp v0.1.11 github.com/projectdiscovery/retryabledns v1.0.21 github.com/projectdiscovery/retryablehttp-go v1.0.14 @@ -38,7 +36,6 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.0 github.com/syndtr/goleveldb v1.0.0 - github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.30.0 github.com/xanzy/go-gitlab v0.82.0 @@ -54,6 +51,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/DataDog/gostackparse v0.6.0 + github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 @@ -69,7 +67,6 @@ require ( github.com/klauspost/compress v1.16.4 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/mitchellh/go-homedir v1.1.0 github.com/projectdiscovery/dsl v0.0.5-0.20230328190851-15d12ab4c5e4 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.8 @@ -82,7 +79,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.0.7 github.com/projectdiscovery/uncover v1.0.2 - github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 + github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a github.com/projectdiscovery/wappalyzergo v0.0.88 github.com/stretchr/testify v1.8.2 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -94,7 +91,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 // indirect @@ -121,6 +117,7 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect github.com/minio/selfupdate v0.6.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -130,6 +127,7 @@ require ( github.com/projectdiscovery/freeport v0.0.4 // indirect github.com/shoenig/go-m1cpu v0.1.4 // indirect github.com/skeema/knownhosts v1.1.0 // indirect + github.com/smartystreets/assertions v1.0.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.2.10 // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -155,7 +153,6 @@ require ( github.com/andybalholm/cascadia v1.3.1 // indirect github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/c4milo/unpackit v0.1.0 // indirect github.com/caddyserver/certmagic v0.17.2 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -174,15 +171,12 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hdm/jarm-go v0.0.7 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/libdns/libdns v0.2.1 // indirect github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect diff --git a/v2/go.sum b/v2/go.sum index d22f6270a7..feb7aef8e5 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -16,8 +16,8 @@ github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1 github.com/DataDog/gostackparse v0.6.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= @@ -63,17 +63,11 @@ github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8 github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= -github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= -github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= -github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= -github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.17.8 h1:GMupCNNI7FARX27L7GjCJM8NgivWbRgpjNI/hOQjFS8= github.com/aws/aws-sdk-go-v2 v1.17.8/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= @@ -116,7 +110,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 h1:Qf1aWwnsNkyAoqDqmdM3nHwN78XQ github.com/aws/aws-sdk-go-v2/service/sts v1.18.9/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= @@ -128,15 +121,9 @@ github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMf github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= -github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= -github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= -github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= github.com/caddyserver/certmagic v0.17.2 h1:o30seC1T/dBqBCNNGNHWwj2i5/I/FMjBbTAhjADP3nE= github.com/caddyserver/certmagic v0.17.2/go.mod h1:ouWUuC490GOLJzkyN35eXfV8bSbwMwSf4bdhkIxtdQE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -171,7 +158,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -193,7 +179,6 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -249,17 +234,12 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= -github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= -github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= -github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -278,8 +258,6 @@ github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= -github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= -github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= @@ -296,12 +274,10 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -320,12 +296,8 @@ github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -355,13 +327,9 @@ github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZj github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -370,7 +338,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/acmez v1.0.4 h1:N3cE4Pek+dSolbsofIkAYz6H1d3pE+2G0os7QHslf80= github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DAYQY= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= @@ -411,7 +378,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= @@ -453,8 +419,8 @@ github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2n github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI= github.com/projectdiscovery/httpx v1.2.9 h1:bSXXjPHIDywokASSXLaWScyIRTFT6Avr9JJS7lV96c0= github.com/projectdiscovery/httpx v1.2.9/go.mod h1:99+TTGdMDKQgWRLzzWj3zGffQ6f4NCAWQHqHWfPn0Uc= -github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d h1:V/TpFiLbpSsgZgxBXSTXSgwGD8uKayImAwqz9etAzK8= -github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d/go.mod h1:6qYsmXQ//CPXib21ugKYXuLwr43zLPlO+UYyx+o+4Mw= +github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467 h1:tMpLB8FzcgAhVOZT9V8G4IYhWufNk/7U5ZqgaVYPeI4= +github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467/go.mod h1:Sk5aRjJk3lYN+MCVIJNxOA6Y+UHgtpyRjKcVqgumFbs= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e1WPXpluc0= @@ -482,8 +448,8 @@ github.com/projectdiscovery/tlsx v1.0.7/go.mod h1:bMz1JMJf1sSBLpk7Y6vLwRZrijxPUv github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 h1:UBOE9Eob1wj7YZ1MGBbtHvc3ptqBHvXxNKRVxTg21Rc= -github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o= +github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a h1:h9ceITnnFLJ0qucXCrI3WOXtNRK2oAtcCuFsMAiuIc0= +github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a/go.mod h1:954dxg9AWmNmcNQdc5BpucghibSvC76prWLQFrv14FQ= github.com/projectdiscovery/wappalyzergo v0.0.88 h1:N/1vFlKmc3GJco9rANJdHrxg8jdav/xmnICo8rztmH8= github.com/projectdiscovery/wappalyzergo v0.0.88/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -494,7 +460,6 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -521,10 +486,8 @@ github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXi github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -569,15 +532,6 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= -github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= -github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= -github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= -github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= -github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= -github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs= -github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -588,7 +542,6 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -660,7 +613,6 @@ golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -727,7 +679,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -845,7 +796,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/v2/internal/installer/doc.go b/v2/internal/installer/doc.go new file mode 100644 index 0000000000..7259db1d62 --- /dev/null +++ b/v2/internal/installer/doc.go @@ -0,0 +1,6 @@ +package installer + +// package install provides helper functions for all download and installation of following tasks: +// 1. downloading and install nuclei templates +// 2. download and update nuclei binary (i.e self update) +// 3. version check for nuclei binary & templates diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go new file mode 100644 index 0000000000..8c0f671a0e --- /dev/null +++ b/v2/internal/installer/template.go @@ -0,0 +1,328 @@ +package installer + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/olekukonko/tablewriter" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + stringsutil "github.com/projectdiscovery/utils/strings" + updateutils "github.com/projectdiscovery/utils/update" +) + +const ( + checkSumFilePerm = 0644 +) + +var ( + HideProgressBar = true + HideUpdateChangesTable = false +) + +// TemplateUpdateResults contains the results of template update +type templateUpdateResults struct { + additions []string + deletions []string + modifications []string + totalCount int +} + +// String returns markdown table of template update results +func (t *templateUpdateResults) String() string { + var buff bytes.Buffer + data := [][]string{ + {strconv.Itoa(t.totalCount), strconv.Itoa(len(t.additions)), strconv.Itoa(len(t.deletions))}, + } + table := tablewriter.NewWriter(&buff) + table.SetHeader([]string{"Total", "Added", "Removed"}) + for _, v := range data { + table.Append(v) + } + table.Render() + return buff.String() +} + +// TemplateManager is a manager for templates. +// It downloads / updates / installs templates. +type TemplateManager struct{} + +// FreshInstallIfNotExists installs templates if they are not already installed +// if templates directory already exists, it does nothing +func (t *TemplateManager) FreshInstallIfNotExists() error { + if fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { + return nil + } + gologger.Info().Msgf("nuclei-templates are not installed, installing...") + return t.installTemplatesAt(config.DefaultConfig.TemplatesDirectory) +} + +// UpdateIfOutdated updates templates if they are outdated +func (t *TemplateManager) UpdateIfOutdated() error { + // if folder does not exist, its a fresh install and not update + if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { + return t.FreshInstallIfNotExists() + } + if config.DefaultConfig.NeedsTemplateUpdate() { + return t.updateTemplatesAt(config.DefaultConfig.TemplatesDirectory) + } + return nil +} + +// installTemplatesAt installs templates at given directory +func (t *TemplateManager) installTemplatesAt(dir string) error { + if !fileutil.FolderExists(dir) { + if err := fileutil.CreateFolder(dir); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to create directory at %s", dir) + } + } + ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTeamplatesRepoName) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) + } + // write templates to disk + if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir) + } + gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir) + return nil +} + +// updateTemplatesAt updates templates at given directory +func (t *TemplateManager) updateTemplatesAt(dir string) error { + // firstly read checksums from .checksum file these are used to generate stats + oldchecksums, err := t.getChecksumFromDir(dir) + if err != nil { + // if something went wrong overwrite all files + oldchecksums = make(map[string]string) + } + + ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTeamplatesRepoName) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) + } + + gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName()) + + // write templates to disk + if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + return err + } + + // get checksums from new templates + newchecksums, err := t.getChecksumFromDir(dir) + if err != nil { + // unlikely this case will happen + return errorutil.NewWithErr(err).Msgf("failed to get checksums from %s after update", dir) + } + + // summarize all changes + results := t.summarizeChanges(oldchecksums, newchecksums) + + // print summary + if results.totalCount > 0 { + gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir) + if !HideUpdateChangesTable { + // print summary table + gologger.Print().Msgf("\nNuclei Templates %s Changelog\n", ghrd.Latest.GetTagName()) + gologger.DefaultLogger.Print().Msg(results.String()) + } + } else { + gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir) + } + return nil +} + +// summarizeChanges summarizes changes between old and new checksums +func (t *TemplateManager) summarizeChanges(old, new map[string]string) *templateUpdateResults { + results := &templateUpdateResults{} + for k, v := range new { + if oldv, ok := old[k]; ok { + if oldv != v { + results.modifications = append(results.modifications, k) + } + } else { + results.additions = append(results.additions, k) + } + } + for k := range old { + if _, ok := new[k]; !ok { + results.deletions = append(results.deletions, k) + } + } + results.totalCount = len(results.additions) + len(results.deletions) + len(results.modifications) + return results +} + +// getAbsoluteFilePath returns absolute path where a file should be written based on given uri(i.e files in zip) +// if returned path is empty, it means that file should not be written and skipped +func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.FileInfo) string { + // overwrite .nuclei-ignore everytime nuclei-templates are downloaded + if f.Name() == config.NucleiIgnoreFileName { + return config.DefaultConfig.GetIgnoreFilePath() + } + // skip all meta files + if !strings.EqualFold(f.Name(), config.NewTemplateAdditionsFileName) { + if strings.TrimSpace(f.Name()) == "" || strings.HasPrefix(f.Name(), ".") || strings.EqualFold(f.Name(), "README.md") { + return "" + } + } + + // get root or leftmost directory name from path + // this is in format `projectdiscovery-nuclei-templates-commithash` + + index := strings.Index(uri, "/") + if index == -1 { + // zip files does not have directory at all , in this case log error but continue + gologger.Warning().Msgf("failed to get directory name from uri: %s", uri) + return filepath.Join(templatedir, uri) + } + // seperator is also included in rootDir + rootDirectory := uri[:index+1] + relPath := strings.TrimPrefix(uri, rootDirectory) + + // if it is a github meta directory skip it + if stringsutil.HasPrefixAny(relPath, ".github", ".git") { + return "" + } + + newPath := filepath.Clean(filepath.Join(templatedir, relPath)) + + if !strings.HasPrefix(newPath, templatedir) { + // we don't allow LFI + return "" + } + + if newPath == templatedir || newPath == templatedir+string(os.PathSeparator) { + // skip writing the folder itself since it already exists + return "" + } + + if relPath != "" && f.IsDir() { + // if uri is a directory, create it + if err := fileutil.CreateFolder(newPath); err != nil { + gologger.Warning().Msgf("uri %v: got %s while installing templates", uri, err) + } + return "" + } + return newPath +} + +// writeChecksumFileInDir is actual method responsible for writing all templates to directory +func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { + callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error { + writePath := t.getAbsoluteFilePath(dir, uri, f) + if writePath == "" { + // skip writing file + return nil + } + bin, err := io.ReadAll(r) + if err != nil { + // if error occurs, iteration also stops + return errorutil.NewWithErr(err).Msgf("failed to read file %s", uri) + } + return os.WriteFile(writePath, bin, f.Mode()) + } + err := ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to download templates") + } + if err := config.DefaultConfig.WriteTemplatesConfig(); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write templates config") + } + // update ignore hash after writing new templates + if err := config.DefaultConfig.UpdateNucleiIgnoreHash(); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to update nuclei ignore hash") + } + + // update templates version in config file + if err := config.DefaultConfig.SetTemplatesVersion(ghrd.Latest.GetTagName()); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to update templates version") + } + + // after installation create and write checksums to .checksum file + return t.writeChecksumFileInDir(dir) +} + +// getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) +// if .checksum file does not exist checksums are calculated and returned +func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, error) { + checksumFilePath := config.DefaultConfig.GetChecksumFilePath() + if fileutil.FileExists(checksumFilePath) { + checksums, err := os.ReadFile(checksumFilePath) + if err == nil { + allChecksums := make(map[string]string) + for _, v := range strings.Split(string(checksums), "\n") { + v = strings.TrimSpace(v) + tmparr := strings.Split(v, ",") + if len(tmparr) != 2 { + continue + } + allChecksums[tmparr[0]] = tmparr[1] + } + return allChecksums, nil + } + } + return t.calculateChecksumMap(dir) +} + +// writeChecksumFileInDir creates checksums of all yaml files in given directory +// and writes them to a file named .checksum +func (t *TemplateManager) writeChecksumFileInDir(dir string) error { + checksumMap, err := t.calculateChecksumMap(dir) + if err != nil { + return err + } + var buff bytes.Buffer + for k, v := range checksumMap { + buff.WriteString(k + "," + v) + } + return os.WriteFile(config.DefaultConfig.GetChecksumFilePath(), buff.Bytes(), checkSumFilePerm) +} + +// getChecksumMap returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) +func (t *TemplateManager) calculateChecksumMap(dir string) (map[string]string, error) { + // getchecksumMap walks given directory `dir` and returns a map containing + // checksums (md5 hash) of all yaml files (with .yaml extension) and the + // format is map[filePath]checksum + checksumMap := map[string]string{} + + getChecksum := func(filepath string) (string, error) { + // return md5 hash of the file + bin, err := os.ReadFile(filepath) + if err != nil { + return "", err + } + return fmt.Sprintf("%x", md5.Sum(bin)), nil + } + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // skip checksums of custom templates i.e github and s3 + if stringsutil.HasPrefixAny(path, config.DefaultConfig.CustomGithubTemplatesDirectory, config.DefaultConfig.CustomS3TemplatesDirectory) { + return nil + } + + // current implementations calculates checksums of all files (including .yaml,.txt,.md,.json etc) + if !d.IsDir() { + checksum, err := getChecksum(path) + if err != nil { + return err + } + checksumMap[path] = checksum + } + return nil + }) + return checksumMap, errorutil.WrapfWithNil(err, "failed to calculate checksums of templates") +} diff --git a/v2/internal/installer/template_test.go b/v2/internal/installer/template_test.go new file mode 100644 index 0000000000..e6b0345e24 --- /dev/null +++ b/v2/internal/installer/template_test.go @@ -0,0 +1,59 @@ +package installer + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/stretchr/testify/require" +) + +func TestTemplateInstallation(t *testing.T) { + // test that the templates are installed correctly + // along with necessary changes that are made + HideProgressBar = true + + tm := &TemplateManager{} + dir, err := os.MkdirTemp("", "nuclei-templates-*") + require.Nil(t, err) + defer os.RemoveAll(dir) + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.Nil(t, err) + defer os.RemoveAll(cfgdir) + + // set the config directory to a temporary directory + config.DefaultConfig.SetConfigDir(cfgdir) + // set the templates directory to a temporary directory + templatesTempDir := filepath.Join(dir, "templates") + config.DefaultConfig.SetTemplatesDir(templatesTempDir) + + err = tm.FreshInstallIfNotExists() + if err != nil { + if strings.Contains(err.Error(), "rate limit") { + t.Skip("Skipping test due to github rate limit") + } + require.Nil(t, err) + } + + // we should switch to more fine granular tests for template + // integrity, but for now, we just check that the templates are installed + counter := 0 + err = filepath.Walk(templatesTempDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + counter++ + } + return nil + }) + require.Nil(t, err) + + // we should have at least 1000 templates + require.Greater(t, counter, 1000) + // everytime we install templates, it should override the ignore file with latest one + require.FileExists(t, config.DefaultConfig.GetIgnoreFilePath()) + t.Logf("Installed %d templates", counter) +} diff --git a/v2/internal/installer/util.go b/v2/internal/installer/util.go new file mode 100644 index 0000000000..7285626687 --- /dev/null +++ b/v2/internal/installer/util.go @@ -0,0 +1,69 @@ +package installer + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + + "github.com/Masterminds/semver/v3" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + errorutil "github.com/projectdiscovery/utils/errors" +) + +// GetNewTemplatesInVersions returns templates path of all newly added templates +// in these versions +func GetNewTemplatesInVersions(versions ...string) []string { + allTemplates := []string{} + for _, v := range versions { + if v == config.DefaultConfig.TemplateVersion { + allTemplates = append(allTemplates, config.DefaultConfig.GetNewAdditions()...) + } + _, err := semver.NewVersion(v) + if err != nil { + gologger.Error().Msgf("%v is not a valid semver version. skipping", v) + continue + } + if config.IsOutdatedVersion(v, "v8.8.4") { + // .new-additions was added in v8.8.4 any version before that is not supported + gologger.Error().Msgf(".new-additions support was added in v8.8.4 older versions are not supported") + continue + } + + arr, err := getNewAdditionsFileFromGithub(v) + if err != nil { + gologger.Error().Msgf("failed to fetch new additions for %v got: %v", v, err) + continue + } + allTemplates = append(allTemplates, arr...) + } + return allTemplates +} + +func getNewAdditionsFileFromGithub(version string) ([]string, error) { + resp, err := retryableHttpClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, errorutil.New("version not found") + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + templatesList := []string{} + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + if config.IsTemplate(text) { + templatesList = append(templatesList, text) + } + } + return templatesList, nil +} diff --git a/v2/internal/installer/versioncheck.go b/v2/internal/installer/versioncheck.go new file mode 100644 index 0000000000..12dd2aed51 --- /dev/null +++ b/v2/internal/installer/versioncheck.go @@ -0,0 +1,91 @@ +package installer + +import ( + "encoding/json" + "io" + "net/url" + "os" + "runtime" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/retryablehttp-go" + updateutils "github.com/projectdiscovery/utils/update" +) + +const ( + pdtmNucleiVersionEndpoint = "https://api.pdtm.sh/api/v1/tools/nuclei" + pdtmNucleiIgnoreFileEndpoint = "https://api.pdtm.sh/api/v1/tools/nuclei/ignore" +) + +// defaultHttpClient is http client that is only meant to be used for version check +// if proxy env variables are set those are reflected in this client +var retryableHttpClient = retryablehttp.NewClient(retryablehttp.Options{HttpClient: updateutils.DefaultHttpClient, RetryMax: 2}) + +// PdtmAPIResponse is the response from pdtm API for nuclei endpoint +type PdtmAPIResponse struct { + IgnoreHash string `json:"ignore-hash"` + Tools []struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"tools"` +} + +// NucleiVersionCheck checks for the latest version of nuclei and nuclei templates +// and returns an error if it fails to check on success it returns nil and changes are +// made to the default config in config.DefaultConfig +func NucleiVersionCheck() error { + resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams()) + if err != nil { + return err + } + defer resp.Body.Close() + bin, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + var pdtmResp PdtmAPIResponse + if err := json.Unmarshal(bin, &pdtmResp); err != nil { + return err + } + var nucleiversion, templateversion string + for _, tool := range pdtmResp.Tools { + switch tool.Name { + case "nuclei": + if tool.Version != "" { + nucleiversion = "v" + tool.Version + } + + case "nuclei-templates": + if tool.Version != "" { + templateversion = "v" + tool.Version + } + } + } + return config.DefaultConfig.WriteVersionCheckData(pdtmResp.IgnoreHash, nucleiversion, templateversion) +} + +// getpdtmParams returns encoded query parameters sent to update check endpoint +func getpdtmParams() string { + params := &url.Values{} + params.Add("os", runtime.GOOS) + params.Add("arch", runtime.GOARCH) + params.Add("go_version", runtime.Version()) + params.Add("v", config.Version) + return params.Encode() +} + +// UpdateIgnoreFile updates default ignore file by downloading latest ignore file +func UpdateIgnoreFile() error { + resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams()) + if err != nil { + return err + } + bin, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil { + return err + } + return config.DefaultConfig.UpdateNucleiIgnoreHash() +} diff --git a/v2/internal/installer/versioncheck_test.go b/v2/internal/installer/versioncheck_test.go new file mode 100644 index 0000000000..dca1665bf7 --- /dev/null +++ b/v2/internal/installer/versioncheck_test.go @@ -0,0 +1,19 @@ +package installer + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/utils/generic" + "github.com/stretchr/testify/require" +) + +func TestVersionCheck(t *testing.T) { + err := NucleiVersionCheck() + require.Nil(t, err) + cfg := config.DefaultConfig + if generic.EqualsAny("", cfg.LatestNucleiIgnoreHash, cfg.LatestNucleiVersion, cfg.LatestNucleiTemplatesVersion) { + // all above values cannot be empty + t.Errorf("something went wrong got empty response nuclei-version=%v templates-version=%v ignore-hash=%v", cfg.LatestNucleiVersion, cfg.LatestNucleiTemplatesVersion, cfg.LatestNucleiIgnoreHash) + } +} diff --git a/v2/internal/installer/zipslip_unix_test.go b/v2/internal/installer/zipslip_unix_test.go new file mode 100644 index 0000000000..7e9eab94a5 --- /dev/null +++ b/v2/internal/installer/zipslip_unix_test.go @@ -0,0 +1,68 @@ +package installer + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var _ fs.FileInfo = &tempFileInfo{} + +type tempFileInfo struct { + name string +} + +func (t *tempFileInfo) Name() string { + return t.name +} + +func (t *tempFileInfo) ModTime() time.Time { + return time.Now() +} + +func (t *tempFileInfo) Mode() fs.FileMode { + return fs.ModePerm +} + +func (t tempFileInfo) IsDir() bool { + return false +} + +func (t *tempFileInfo) Size() int64 { + return 100 +} + +func (t *tempFileInfo) Sys() any { + return nil +} + +func TestZipSlip(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping Unix Zip LFI Check") + } + + configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") + defer os.RemoveAll(configuredTemplateDirectory) + + t.Run("negative scenarios", func(t *testing.T) { + filePathsFromZip := []string{ + "./../nuclei-templates/../cve/test.yaml", + "nuclei-templates/../cve/test.yaml", + "nuclei-templates/././../cve/test.yaml", + "nuclei-templates/.././../cve/test.yaml", + "nuclei-templates/.././../cve/../test.yaml", + } + tm := TemplateManager{} + + for _, filePathFromZip := range filePathsFromZip { + var tmp fs.FileInfo = &tempFileInfo{name: filePathFromZip} + writePath := tm.getAbsoluteFilePath(configuredTemplateDirectory, filePathFromZip, tmp) + require.Equal(t, "", writePath, filePathFromZip) + } + }) +} diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 6054fcea97..03725ec180 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -5,6 +5,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + updateutils "github.com/projectdiscovery/utils/update" ) var banner = fmt.Sprintf(` @@ -12,7 +13,7 @@ var banner = fmt.Sprintf(` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / -/_/ /_/\__,_/\___/_/\___/_/ v%s +/_/ /_/\__,_/\___/_/\___/_/ %s `, config.Version) // showBanner is used to show the banner to the user @@ -20,3 +21,9 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") } + +// NucleiToolUpdateCallback updates nuclei binary/tool to latest version +func NucleiToolUpdateCallback() { + showBanner() + updateutils.GetUpdateToolCallback("nuclei", config.Version)() +} diff --git a/v2/internal/runner/healthcheck.go b/v2/internal/runner/healthcheck.go index 7c10b6df98..abeb51ecc0 100644 --- a/v2/internal/runner/healthcheck.go +++ b/v2/internal/runner/healthcheck.go @@ -3,7 +3,6 @@ package runner import ( "fmt" "net" - "path/filepath" "runtime" "strings" @@ -23,15 +22,8 @@ func DoHealthCheck(options *types.Options) string { test.WriteString(fmt.Sprintf("Compiler: %s\n", runtime.Compiler)) var testResult string - - nucleiIgnorePath := config.GetIgnoreFilePath() - cf, _ := config.ReadConfiguration() - templatePath := "" - if cf != nil { - templatePath = cf.TemplatesDirectory - } - nucleiTemplatePath := filepath.Join(templatePath, "/", ".checksum") - for _, filename := range []string{options.ConfigPath, nucleiIgnorePath, nucleiTemplatePath} { + cfg := config.DefaultConfig + for _, filename := range []string{cfg.GetFlagsConfigFilePath(), cfg.GetIgnoreFilePath(), cfg.GetChecksumFilePath()} { ok, err := fileutil.IsReadable(filename) if ok { testResult = "Ok" diff --git a/v2/internal/runner/nucleicloud/utils.go b/v2/internal/runner/nucleicloud/utils.go index 77a87573d5..5586937cac 100644 --- a/v2/internal/runner/nucleicloud/utils.go +++ b/v2/internal/runner/nucleicloud/utils.go @@ -14,10 +14,7 @@ const DDMMYYYYhhmmss = "2006-01-02 15:04:05" // ReadCatalogChecksum reads catalog checksum from nuclei-templates repository func ReadCatalogChecksum() map[string]string { - config, _ := config.ReadConfiguration() - if config == nil { - return nil - } + config := config.DefaultConfig checksumFile := filepath.Join(config.TemplatesDirectory, "templates-checksum.txt") file, err := os.Open(checksumFile) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 3d880e125a..7a89f7de35 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -26,8 +26,10 @@ import ( ) func ConfigureOptions() error { + // with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions + // if file has extension `.yaml,.json` we consider those as strings and not files to be read isFromFileFunc := func(s string) bool { - return !isTemplate(s) + return !config.IsTemplate(s) } goflags.FileNormalizedStringSliceOptions.IsFromFile = isFromFileFunc goflags.FileStringSliceOptions.IsFromFile = isFromFileFunc @@ -48,31 +50,9 @@ func ParseOptions(options *types.Options) { // Show the user the banner showBanner() - if options.TemplatesDirectory != "" && !filepath.IsAbs(options.TemplatesDirectory) { - cwd, _ := os.Getwd() - options.TemplatesDirectory = filepath.Join(cwd, options.TemplatesDirectory) - } - if options.Version { - gologger.Info().Msgf("Current Version: %s\n", config.Version) - os.Exit(0) - } if options.ShowVarDump { vardump.EnableVarDump = true } - if options.TemplatesVersion { - configuration, err := config.ReadConfiguration() - if err != nil { - gologger.Fatal().Msgf("Could not read template configuration: %s\n", err) - } - gologger.Info().Msgf("Public nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory) - if configuration.CustomS3TemplatesDirectory != "" { - gologger.Info().Msgf("Custom S3 templates location: %s\n", configuration.CustomS3TemplatesDirectory) - } - if configuration.CustomGithubTemplatesDirectory != "" { - gologger.Info().Msgf("Custom Github templates location: %s ", configuration.CustomGithubTemplatesDirectory) - } - os.Exit(0) - } if options.ShowActions { gologger.Info().Msgf("Showing available headless actions: ") for action := range engine.ActionStringToAction { @@ -139,7 +119,7 @@ func validateOptions(options *types.Options) error { return err } if options.Validate { - validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) + validateTemplatePaths(config.DefaultConfig.TemplatesDirectory, options.Templates, options.Workflows) } // Verify if any of the client certificate options were set since it requires all three to work properly diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index ba467d159a..54039e909d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -1,25 +1,21 @@ package runner import ( - "bufio" - "bytes" "context" "encoding/json" - "fmt" - "io" "net/http" _ "net/http/pprof" "os" - "path/filepath" "reflect" "strconv" "strings" "sync/atomic" "time" + "github.com/projectdiscovery/nuclei/v2/internal/installer" "github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud" + updateutils "github.com/projectdiscovery/utils/update" - "github.com/blang/semver" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/ratelimit" @@ -59,14 +55,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml" "github.com/projectdiscovery/retryablehttp-go" - stringsutil "github.com/projectdiscovery/utils/strings" ) // Runner is a client for running the enumeration process. type Runner struct { output output.Writer interactsh *interactsh.Client - templatesConfig *config.Config options *types.Options projectFile *projectfile.ProjectFile catalog catalog.Catalog @@ -101,16 +95,31 @@ func New(options *types.Options) (*Runner, error) { runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey) } - if options.UpdateNuclei { - if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil { - return nil, err + // Version check by default + if config.DefaultConfig.CanCheckForUpdates() { + if err := installer.NucleiVersionCheck(); err != nil { + if options.Verbose || options.Debug { + gologger.Error().Msgf("nuclei version check failed got: %s\n", err) + } + } + // Check for template updates and update if available + tm := &installer.TemplateManager{} + if err := tm.FreshInstallIfNotExists(); err != nil { + gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err) + } + if err := tm.UpdateIfOutdated(); err != nil { + gologger.Warning().Msgf("failed to update nuclei templates: %s\n", err) + } + + if config.DefaultConfig.NeedsIgnoreFileUpdate() { + if err := installer.UpdateIgnoreFile(); err != nil { + gologger.Warning().Msgf("failed to update nuclei ignore file: %s\n", err) + } } - return nil, nil } + if options.Validate { parsers.ShouldValidate = true - // Does not update the templates when validate flag is used - options.NoUpdateTemplates = true } // TODO: refactor to pass options reference globally without cycles @@ -119,9 +128,6 @@ func New(options *types.Options) (*Runner, error) { // parse the runner.options.GithubTemplateRepo and store the valid repos in runner.customTemplateRepos runner.customTemplates = customtemplates.ParseCustomTemplates(runner.options) - if err := runner.updateTemplates(); err != nil { - gologger.Error().Msgf("Could not update templates: %s\n", err) - } if options.Headless { if engine.MustDisableSandbox() { gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n") @@ -133,7 +139,7 @@ func New(options *types.Options) (*Runner, error) { runner.browser = browser } - runner.catalog = disk.NewCatalog(runner.options.TemplatesDirectory) + runner.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) var httpclient *retryablehttp.Client if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { @@ -374,34 +380,13 @@ func (r *Runner) Close() { func (r *Runner) RunEnumeration() error { // If user asked for new templates to be executed, collect the list from the templates' directory. if r.options.NewTemplates { - templatesLoaded, err := r.readNewTemplatesFile() - if err != nil { - return errors.Wrap(err, "could not get newly added templates") + if arr := config.DefaultConfig.GetNewAdditions(); len(arr) > 0 { + r.options.Templates = append(r.options.Templates, arr...) } - r.options.Templates = append(r.options.Templates, templatesLoaded...) } if len(r.options.NewTemplatesWithVersion) > 0 { - minVersion, err := semver.Parse("8.8.4") - if err != nil { - return errors.Wrap(err, "could not parse minimum version") - } - latestVersion, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return errors.Wrap(err, "could not get latest version") - } - for _, version := range r.options.NewTemplatesWithVersion { - current, err := semver.Parse(strings.Trim(version, "v")) - if err != nil { - return errors.Wrap(err, "could not parse current version") - } - if !(current.GT(minVersion) && current.LTE(latestVersion)) { - return fmt.Errorf("version should be greater than %s and less than %s", minVersion, latestVersion) - } - templatesLoaded, err := r.readNewTemplatesWithVersionFile(fmt.Sprintf("v%s", current)) - if err != nil { - return errors.Wrap(err, "could not get newly added templates for "+current.String()) - } - r.options.Templates = append(r.options.Templates, templatesLoaded...) + if arr := installer.GetNewTemplatesInVersions(r.options.NewTemplatesWithVersion...); len(arr) > 0 { + r.options.Templates = append(r.options.Templates, arr...) } } // Exclude ignored file for validation @@ -445,12 +430,7 @@ func (r *Runner) RunEnumeration() error { } executerOpts.WorkflowLoader = workflowLoader - templateConfig := r.templatesConfig - if templateConfig == nil { - templateConfig = &config.Config{} - } - - store, err := loader.New(loader.NewConfig(r.options, templateConfig, r.catalog, executerOpts)) + store, err := loader.New(loader.NewConfig(r.options, r.catalog, executerOpts)) if err != nil { return errors.Wrap(err, "could not load templates from config") } @@ -499,6 +479,8 @@ func (r *Runner) RunEnumeration() error { r.listAvailableStoreTemplates(store) os.Exit(0) } + + // display execution info like version , templates used etc r.displayExecutionInfo(store) // If not explicitly disabled, check if http based protocols @@ -685,136 +667,21 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.Display(parsers.SyntaxErrorStats) stats.Display(parsers.RuntimeWarningsStats) - builder := &strings.Builder{} - if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { - builder.WriteString(" (") + cfg := config.DefaultConfig - if strings.Contains(config.Version, "-dev") { - builder.WriteString(r.colorizer.Blue("development").String()) - } else if config.Version == r.templatesConfig.NucleiLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr := builder.String() - builder.Reset() - - gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) + gologger.Info().Msgf("Current nuclei version: %v %v", config.Version, updateutils.GetVersionDescription(config.Version, cfg.LatestNucleiVersion)) + gologger.Info().Msgf("Current nuclei-templates version: %v %v", cfg.TemplateVersion, updateutils.GetVersionDescription(cfg.TemplateVersion, cfg.LatestNucleiTemplatesVersion)) - if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic - builder.WriteString(" (") - - if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr = builder.String() - builder.Reset() - - if r.templatesConfig != nil { - gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) - } if len(store.Templates()) > 0 { - gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) - gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) + gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) + gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates())) } if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) + gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) } if r.hmapInputProvider.Count() > 0 { - gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count()) - } -} - -func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) { - resp, err := retryablehttp.DefaultClient().Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version)) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, errors.New("version not found") - } - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - templatesList := []string{} - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if isTemplate(text) { - templatesList = append(templatesList, text) - } - } - return templatesList, nil -} - -// readNewTemplatesFile reads newly added templates from directory if it exists -func (r *Runner) readNewTemplatesFile() ([]string, error) { - if r.templatesConfig == nil { - return nil, nil - } - additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions") - file, err := os.Open(additionsFile) - if err != nil { - return nil, err - } - defer file.Close() - - templatesList := []string{} - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if isTemplate(text) { - templatesList = append(templatesList, text) - } + gologger.Info().Msgf("Targets loaded for current scan: %d", r.hmapInputProvider.Count()) } - return templatesList, nil -} - -// countNewTemplates returns the number of newly added templates -func (r *Runner) countNewTemplates() int { - if r.templatesConfig == nil { - return 0 - } - additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions") - file, err := os.Open(additionsFile) - if err != nil { - return 0 - } - defer file.Close() - - count := 0 - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - - if isTemplate(text) { - count++ - } - - } - return count -} - -// isTemplate is a callback function used by goflags to decide if given file should be read -// if it is not a nuclei-template file only then file is read -func isTemplate(filename string) bool { - return stringsutil.EqualFoldAny(filepath.Ext(filename), config.GetSupportTemplateFileExtensions()...) } // SaveResumeConfig to file diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 4c61439079..de667f7452 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -8,6 +8,7 @@ import ( "github.com/alecthomas/chroma/quick" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/gologger" @@ -36,9 +37,9 @@ func (r *Runner) verboseTemplate(tpl *templates.Template) { func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { gologger.Print().Msgf( - "\nListing available v.%s nuclei templates for %s", - r.templatesConfig.TemplateVersion, - r.templatesConfig.TemplatesDirectory, + "\nListing available %v nuclei templates for %v", + config.DefaultConfig.TemplateVersion, + config.DefaultConfig.TemplatesDirectory, ) for _, tpl := range store.Templates() { if hasExtraFlags(r.options) { @@ -63,7 +64,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { } gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody) } else { - gologger.Silent().Msgf("%s\n", strings.TrimPrefix(tpl.Path, r.templatesConfig.TemplatesDirectory+string(filepath.Separator))) + gologger.Silent().Msgf("%s\n", strings.TrimPrefix(tpl.Path, config.DefaultConfig.TemplatesDirectory+string(filepath.Separator))) } } else { r.verboseTemplate(tpl) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go deleted file mode 100644 index 8cf463b7c8..0000000000 --- a/v2/internal/runner/update.go +++ /dev/null @@ -1,646 +0,0 @@ -package runner - -import ( - "archive/zip" - "bufio" - "bytes" - "context" - "crypto/md5" - "encoding/hex" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" - - "github.com/apex/log" - "github.com/blang/semver" - "github.com/google/go-github/github" - "github.com/olekukonko/tablewriter" - "github.com/pkg/errors" - "golang.org/x/oauth2" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v2/pkg/external/customtemplates" - client "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/updatecheck" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" - "github.com/projectdiscovery/retryablehttp-go" - fileutil "github.com/projectdiscovery/utils/file" - folderutil "github.com/projectdiscovery/utils/folder" - - "github.com/tj/go-update" - "github.com/tj/go-update/progress" - githubUpdateStore "github.com/tj/go-update/stores/github" -) - -const ( - userName = "projectdiscovery" - repoName = "nuclei-templates" - nucleiIgnoreFile = ".nuclei-ignore" - nucleiConfigFilename = ".templates-config.json" -) - -var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) - -// updateTemplates checks if the default list of nuclei-templates -// exist in the user's home directory, if not the latest revision -// is downloaded from GitHub. -// -// If the path exists but does not contain the latest version of public templates, -// the new version is downloaded from GitHub to the templates' directory, overwriting the old content. -func (r *Runner) updateTemplates() error { // TODO this method does more than just update templates. Should be refactored. - configDir, err := config.GetConfigDir() - if err != nil { - return err - } - _ = os.MkdirAll(configDir, 0755) - - if err := r.readInternalConfigurationFile(configDir); err != nil { - return errors.Wrap(err, "could not read configuration file") - } - - // If the config doesn't exist, create it now. - defaultTemplatesDirectory, err := utils.GetDefaultTemplatePath() - if err != nil { - return err - } - err = r.createDefaultConfig(defaultTemplatesDirectory) - if err != nil { - return err - } - - if r.options.TemplatesDirectory == "" { - // if no -tud flag passed then read from template config - if r.templatesConfig.TemplatesDirectory != "" { - r.options.TemplatesDirectory = r.templatesConfig.TemplatesDirectory - } else { - r.options.TemplatesDirectory = defaultTemplatesDirectory - } - } else if r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory { - // if -tud pass then update the templateConfig & it is diff then template config - r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory) - } - - // if disable update check flag is passed and no update template flag is passed - if (r.options.NoUpdateTemplates && !r.options.UpdateTemplates) || r.options.Cloud { - return nil - } - - client.InitNucleiVersion(config.Version) - r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions - - ctx := context.Background() - - var noTemplatesFound bool - if !fileutil.FolderExists(r.templatesConfig.TemplatesDirectory) { - noTemplatesFound = true - } - if r.templatesConfig.TemplateVersion == "" || noTemplatesFound { - return r.freshTemplateInstallation(configDir, ctx) - } - - // download | update the custom templates repos - if r.options.UpdateTemplates { - for _, ct := range r.customTemplates { - ct.Update(r.templatesConfig.TemplatesDirectory, ctx) - } - } - - latestVersion, currentVersion, err := getVersions(r) - if err != nil { - return err - } - - if latestVersion.EQ(currentVersion) { - if r.options.UpdateTemplates { - gologger.Info().Msgf("No new updates found for nuclei templates") - } - return config.WriteConfiguration(r.templatesConfig) - } - - if err := r.updateTemplatesWithVersion(latestVersion, currentVersion, r, ctx); err != nil { - return err - } - return nil -} - -// createDefaultConfig create template config file is template config is not found -func (r *Runner) createDefaultConfig(defaultTemplatesDirectory string) error { - // TODO remove customTemplate check in next version. - if r.templatesConfig == nil || r.templatesConfig.CustomGithubTemplatesDirectory == "" || r.templatesConfig.CustomS3TemplatesDirectory == "" { - currentConfig := &config.Config{ - TemplatesDirectory: defaultTemplatesDirectory, - NucleiVersion: config.Version, - CustomS3TemplatesDirectory: filepath.Join(defaultTemplatesDirectory, customtemplates.CustomS3TemplateDirectory), - CustomGithubTemplatesDirectory: filepath.Join(defaultTemplatesDirectory, customtemplates.CustomGithubTemplateDirectory), - } - r.templatesConfig = currentConfig - if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil { - return errors.Wrap(writeErr, "could not write template configuration") - } - } - return nil -} - -// freshTemplateInstallation downloads the nuclei template and custom templates if new directory passed -func (r *Runner) freshTemplateInstallation(configDir string, ctx context.Context) error { - gologger.Info().Msgf("nuclei-templates are not installed, installing...\n") - - r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions - - version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - - // Download the repository and write the revision to a HEAD file. - asset, getErr := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion) - if getErr != nil { - return getErr - } - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - - if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { - return err - } - r.templatesConfig.TemplateVersion = version.String() - - if err := config.WriteConfiguration(r.templatesConfig); err != nil { - return err - } - gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory) - - // case where -gtr flag is passed for the first time installation - for _, ct := range r.customTemplates { - ct.Download(r.templatesConfig.TemplatesDirectory, ctx) - } - return nil -} - -func (r *Runner) updateTemplatesWithVersion(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { - if latestVersion.GT(currentVersion) { - gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String()) - gologger.Info().Msgf("Downloading latest release...") - - if runner.options.TemplatesDirectory != "" { - runner.templatesConfig.TemplatesDirectory = runner.options.TemplatesDirectory - } - runner.templatesConfig.TemplateVersion = latestVersion.String() - - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", latestVersion.String(), runner.templatesConfig.TemplatesDirectory) - - asset, err := runner.getLatestReleaseFromGithub(runner.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - if _, err := runner.downloadReleaseAndUnzip(ctx, latestVersion.String(), asset.GetZipballURL()); err != nil { - return err - } - if err := config.WriteConfiguration(runner.templatesConfig); err != nil { - return err - } - gologger.Info().Msgf("Successfully updated nuclei-templates (v%s) to %s. GoodLuck!\n", latestVersion.String(), r.templatesConfig.TemplatesDirectory) - } - return nil -} - -func getVersions(runner *Runner) (semver.Version, semver.Version, error) { - // Get the configuration currently on disk. - verText := runner.templatesConfig.TemplateVersion - indices := reVersion.FindStringIndex(verText) - if indices == nil { - return semver.Version{}, semver.Version{}, fmt.Errorf("invalid release found with tag %s", verText) - } - if indices[0] > 0 { - verText = verText[indices[0]:] - } - - currentVersion, err := semver.Make(verText) - if err != nil { - return semver.Version{}, semver.Version{}, err - } - - latestVersion, err := semver.Parse(runner.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return semver.Version{}, semver.Version{}, err - } - return latestVersion, currentVersion, nil -} - -// readInternalConfigurationFile reads the internal configuration file for nuclei -func (r *Runner) readInternalConfigurationFile(configDir string) error { - templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) - if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { - configuration, readErr := config.ReadConfiguration() - if readErr != nil { - return readErr - } - r.templatesConfig = configuration - } - return nil -} - -// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from GitHub -func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { - data, err := client.GetLatestIgnoreFile() - if err != nil { - return false - } - if len(data) > 0 { - _ = os.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644) - } - if r.templatesConfig != nil { - if err := config.WriteConfiguration(r.templatesConfig); err != nil { - gologger.Warning().Msgf("Could not get ignore-file from server: %s", err) - } - } - return true -} - -func getGHClientIncognito() *github.Client { - var tc *http.Client - return github.NewClient(tc) -} - -func getGHClientWithToken() *github.Client { - if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok { - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - oauthClient := oauth2.NewClient(ctx, ts) - return github.NewClient(oauthClient) - } - return nil -} - -// getLatestReleaseFromGithub returns the latest release from GitHub -func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) { - var ( - gitHubClient *github.Client - retried bool - ) - gitHubClient = getGHClientIncognito() -getRelease: - release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) - if err != nil { - // retry with authentication - if gitHubClient = getGHClientWithToken(); gitHubClient != nil && !retried { - retried = true - goto getRelease - } - return nil, err - } - if release == nil { - return nil, errors.New("no version found for the templates") - } - return release, nil -} - -// downloadReleaseAndUnzip downloads and unzips the release in a directory -func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP request to %s: %w", downloadURL, err) - } - - res, err := retryablehttp.DefaultClient().Do(req) - if err != nil { - return nil, fmt.Errorf("failed to download a release file from %s: %w", downloadURL, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode) - } - - buf, err := io.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("failed to create buffer for zip file: %w", err) - } - - reader := bytes.NewReader(buf) - zipReader, err := zip.NewReader(reader, reader.Size()) - if err != nil { - return nil, fmt.Errorf("failed to uncompress zip file: %w", err) - } - - // Create the template folder if it doesn't exist - if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create template base folder: %w", err) - } - - results, err := r.compareAndWriteTemplates(zipReader) - if err != nil { - return nil, fmt.Errorf("failed to write templates: %w", err) - } - - if r.options.Verbose { - r.printUpdateChangelog(results, version) - } - checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum") - if err := writeTemplatesChecksum(checksumFile, results.checksums); err != nil { - return nil, errors.Wrap(err, "could not write checksum") - } - - return results, err -} - -type templateUpdateResults struct { - additions []string - deletions []string - modifications []string - totalCount int - checksums map[string]string -} - -// compareAndWriteTemplates compares and returns the stats of a template update operations. -func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdateResults, error) { - results := &templateUpdateResults{ - checksums: make(map[string]string), - } - - // We use file-checksums that are md5 hashes to store the list of files->hashes - // that have been downloaded previously. - // If the path isn't found in new update after being read from the previous checksum, - // it is removed. This allows us fine-grained control over the download process - // as well as solves a long problem with nuclei-template updates. - configuredTemplateDirectory := r.templatesConfig.TemplatesDirectory - checksumFile := filepath.Join(configuredTemplateDirectory, ".checksum") - templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile) - for _, zipTemplateFile := range zipReader.File { - templateAbsolutePath, skipFile, err := calculateTemplateAbsolutePath(zipTemplateFile.Name, configuredTemplateDirectory) - if err != nil { - return nil, err - } - if skipFile { - continue - } - - newTemplateChecksum, err := writeUnZippedTemplateFile(templateAbsolutePath, zipTemplateFile) - if err != nil { - return nil, err - } - - oldTemplateChecksum, checksumOk := templateChecksumsMap[templateAbsolutePath] - - relativeTemplatePath, err := filepath.Rel(configuredTemplateDirectory, templateAbsolutePath) - if err != nil { - return nil, fmt.Errorf("could not calculate relative path for template: %s. %w", templateAbsolutePath, err) - } - - if checksumOk && oldTemplateChecksum[0] != newTemplateChecksum { - results.modifications = append(results.modifications, relativeTemplatePath) - } - results.checksums[templateAbsolutePath] = newTemplateChecksum - results.totalCount++ - } - - var err error - results.additions, err = r.readNewTemplatesFile() - if err != nil { - results.additions = []string{} - } - - // If we don't find the previous file in the newly downloaded list, - // and it hasn't been changed on the disk, delete it. - for templatePath, templateChecksums := range templateChecksumsMap { - _, ok := results.checksums[templatePath] - if !ok && templateChecksums[0] == templateChecksums[1] { - _ = os.Remove(templatePath) - results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, configuredTemplateDirectory), string(os.PathSeparator))) - } - } - return results, nil -} - -func writeUnZippedTemplateFile(templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { - templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return "", fmt.Errorf("could not create template file: %w", err) - } - - zipTemplateFileReader, err := zipTemplateFile.Open() - if err != nil { - _ = templateFile.Close() - return "", fmt.Errorf("could not open archive to extract file: %w", err) - } - - md5Hash := md5.New() - - // Save file and also read into hash.Hash for md5 - if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, md5Hash)); err != nil { - _ = templateFile.Close() - return "", fmt.Errorf("could not write template file: %w", err) - } - - if err := templateFile.Close(); err != nil { - return "", fmt.Errorf("could not close file newly created template file: %w", err) - } - - checksum := hex.EncodeToString(md5Hash.Sum(nil)) - return checksum, nil -} - -func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory string) (string, bool, error) { - directory, fileName := filepath.Split(zipFilePath) - - // overwrite .nuclei-ignore everytime nuclei-templates are downloaded - if fileName == ".nuclei-ignore" { - return config.GetIgnoreFilePath(), false, nil - } - - if !strings.EqualFold(fileName, ".new-additions") { - if strings.TrimSpace(fileName) == "" || strings.HasPrefix(fileName, ".") || strings.EqualFold(fileName, "README.md") { - return "", true, nil - } - } - - var ( - directoryPathChunks []string - relativeDirectoryPathWithoutZipRoot string - ) - if folderutil.IsUnixOS() { - directoryPathChunks = strings.Split(directory, string(os.PathSeparator)) - } else if folderutil.IsWindowsOS() { - pathInfo, _ := folderutil.NewPathInfo(directory) - directoryPathChunks = pathInfo.Parts - } - relativeDirectoryPathWithoutZipRoot = filepath.Join(directoryPathChunks[1:]...) - - if strings.HasPrefix(relativeDirectoryPathWithoutZipRoot, ".") { - return "", true, nil - } - - templateDirectory := filepath.Join(configuredTemplateDirectory, relativeDirectoryPathWithoutZipRoot) - - if err := os.MkdirAll(templateDirectory, 0755); err != nil { - return "", false, fmt.Errorf("failed to create template folder: %s. %w", templateDirectory, err) - } - - return filepath.Join(templateDirectory, fileName), false, nil -} - -// createTemplateChecksumsMap reads the previous checksum file from the disk. -// Creates a map of template paths and their previous and currently calculated checksums as values. -func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) { - checksumFile, err := os.Open(checksumsFilePath) - if err != nil { - return nil, err - } - defer checksumFile.Close() - scanner := bufio.NewScanner(checksumFile) - - templatePathChecksumsMap := make(map[string][2]string) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - - parts := strings.Split(text, ",") - if len(parts) < 2 { - continue - } - templatePath := parts[0] - expectedTemplateChecksum := parts[1] - - templateFile, err := os.Open(templatePath) - if err != nil { - return nil, err - } - - hasher := md5.New() - if _, err := io.Copy(hasher, templateFile); err != nil { - return nil, err - } - templateFile.Close() - - values := [2]string{expectedTemplateChecksum} - values[1] = hex.EncodeToString(hasher.Sum(nil)) - templatePathChecksumsMap[templatePath] = values - } - return templatePathChecksumsMap, nil -} - -// writeTemplatesChecksum writes the nuclei-templates checksum data to disk. -func writeTemplatesChecksum(file string, checksum map[string]string) error { - f, err := os.Create(file) - if err != nil { - return err - } - defer f.Close() - - builder := &strings.Builder{} - for k, v := range checksum { - builder.WriteString(k) - builder.WriteString(",") - builder.WriteString(v) - builder.WriteString("\n") - - if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil { - return err - } - builder.Reset() - } - return nil -} - -func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) { - if len(results.additions) > 0 && r.options.Verbose { - gologger.Print().Msgf("\nNewly added templates: \n\n") - - for _, addition := range results.additions { - gologger.Print().Msgf("%s", addition) - } - } - - gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version) - data := [][]string{ - {strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))}, - } - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Total", "Added", "Removed"}) - for _, v := range data { - table.Append(v) - } - table.Render() -} - -// fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub -// -// This fetches the latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions -// If you want to disable this automatic update check, use -nut flag. -func (r *Runner) fetchLatestVersionsFromGithub(configDir string) { - versions, err := client.GetLatestNucleiTemplatesVersion() - if err != nil { - gologger.Warning().Msgf("Could not fetch latest releases: %s", err) - return - } - if r.templatesConfig != nil { - r.templatesConfig.NucleiLatestVersion = versions.Nuclei - r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates - - // If the fetch has resulted in new version of ignore file, update. - if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash { - r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash - r.checkNucleiIgnoreFileUpdates(configDir) - } - } -} - -// updateNucleiVersionToLatest implements nuclei auto-update using GitHub Releases. -func updateNucleiVersionToLatest(verbose bool) error { - if verbose { - log.SetLevel(log.DebugLevel) - } - var command string - switch runtime.GOOS { - case "windows": - command = "nuclei.exe" - default: - command = "nuclei" - } - m := &update.Manager{ - Command: command, - Store: &githubUpdateStore.Store{ - Owner: "projectdiscovery", - Repo: "nuclei", - Version: config.Version, - }, - } - releases, err := m.LatestReleases() - if err != nil { - return errors.Wrap(err, "could not fetch latest release") - } - if len(releases) == 0 { - gologger.Info().Msgf("No new updates found for nuclei engine!") - return nil - } - - latest := releases[0] - var currentOS string - switch runtime.GOOS { - case "darwin": - currentOS = "macOS" - default: - currentOS = runtime.GOOS - } - final := latest.FindZip(currentOS, runtime.GOARCH) - if final == nil { - return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH) - } - tarball, err := final.DownloadProxy(progress.Reader) - if err != nil { - return errors.Wrap(err, "could not download latest release") - } - if err := m.Install(tarball); err != nil { - return errors.Wrap(err, "could not install latest release") - } - gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version) - return nil -} diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go deleted file mode 100644 index 1a8191de36..0000000000 --- a/v2/internal/runner/update_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package runner - -import ( - "archive/zip" - "context" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v2/pkg/testutils" -) - -func TestDownloadReleaseAndUnzipAddition(t *testing.T) { - gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) - - templatesDirectory, err := os.MkdirTemp("", "template-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(templatesDirectory) - - r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions} - - newTempDir, err := os.MkdirTemp("", "new-tmp-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(newTempDir) - - err = os.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), os.ModePerm) - require.Nil(t, err, "could not create base file") - err = os.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), os.ModePerm) - require.Nil(t, err, "could not create new file") - err = os.WriteFile(filepath.Join(newTempDir, ".new-additions"), []byte("new.yaml"), os.ModePerm) - require.Nil(t, err, "could not create new file") - - err = zipFromDirectory("new.zip", newTempDir) - require.Nil(t, err, "could not create new zip from directory") - defer os.Remove("new.zip") - - ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, "new.zip") - })) - defer ts2.Close() - - results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL) - require.Nil(t, err, "could not download release and unzip") - - require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition") -} - -func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { - gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) - - baseTemplates, err := os.MkdirTemp("", "old-temp-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(baseTemplates) - - err = os.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), os.ModePerm) - require.Nil(t, err, "could not create write base file") - err = os.WriteFile(filepath.Join(baseTemplates, ".new-additions"), []byte("base.yaml"), os.ModePerm) - require.Nil(t, err, "could not create new file") - - err = zipFromDirectory("base.zip", baseTemplates) - require.Nil(t, err, "could not create zip from directory") - defer os.Remove("base.zip") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, "base.zip") - })) - defer ts.Close() - - templatesDirectory, err := os.MkdirTemp("", "template-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(templatesDirectory) - - r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions} - - results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) - require.Nil(t, err, "could not download release and unzip") - require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition") - - newTempDir, err := os.MkdirTemp("", "new-tmp-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(newTempDir) - - err = os.WriteFile(filepath.Join(newTempDir, ".new-additions"), []byte(""), os.ModePerm) - require.Nil(t, err, "could not create new file") - - err = zipFromDirectory("new.zip", newTempDir) - require.Nil(t, err, "could not create new zip from directory") - defer os.Remove("new.zip") - - ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, "new.zip") - })) - defer ts2.Close() - - results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL) - require.Nil(t, err, "could not download release and unzip") - - require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions") -} - -func TestCalculateTemplateAbsolutePathPositiveScenario(t *testing.T) { - configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") - defer os.RemoveAll(configuredTemplateDirectory) - - t.Run("positive scenarios", func(t *testing.T) { - zipFilePathsExpectedPathsMap := map[string]string{ - "nuclei-templates/cve/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test.yaml"), - "nuclei-templates/cve/test/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test/test.yaml"), - } - - for filePathFromZip, expectedTemplateAbsPath := range zipFilePathsExpectedPathsMap { - calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) - require.Nil(t, err) - require.Equal(t, expectedTemplateAbsPath, calculatedTemplateAbsPath) - require.False(t, skipFile) - } - }) -} - -func zipFromDirectory(zipPath, directory string) error { - file, err := os.Create(zipPath) - if err != nil { - return err - } - defer file.Close() - - w := zip.NewWriter(file) - defer w.Close() - - walker := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - f, err := w.Create(strings.TrimPrefix(path, directory)) - if err != nil { - return err - } - _, err = io.Copy(f, file) - if err != nil { - return err - } - return nil - } - return filepath.Walk(directory, walker) -} diff --git a/v2/internal/runner/update_unix_test.go b/v2/internal/runner/update_unix_test.go deleted file mode 100644 index c9e1a9a2d3..0000000000 --- a/v2/internal/runner/update_unix_test.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build !windows - -package runner - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCalculateTemplateAbsolutePathNegativeScenario(t *testing.T) { - configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") - defer os.RemoveAll(configuredTemplateDirectory) - - t.Run("negative scenarios", func(t *testing.T) { - filePathsFromZip := []string{ - "./../nuclei-templates/../cve/test.yaml", - "nuclei-templates/../cve/test.yaml", - "nuclei-templates/cve/../test.yaml", - "nuclei-templates/././../cve/test.yaml", - "nuclei-templates/.././../cve/test.yaml", - "nuclei-templates/.././../cve/../test.yaml", - } - - for _, filePathFromZip := range filePathsFromZip { - calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) - require.Nil(t, err) - require.True(t, skipFile) - require.Equal(t, "", calculatedTemplateAbsPath) - } - }) -} diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go deleted file mode 100644 index 44e3141acb..0000000000 --- a/v2/pkg/catalog/config/config.go +++ /dev/null @@ -1,183 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - - jsoniter "github.com/json-iterator/go" - "github.com/mitchellh/go-homedir" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" - - "github.com/projectdiscovery/gologger" - fileutil "github.com/projectdiscovery/utils/file" -) - -// Config contains the internal nuclei engine configuration -type Config struct { - TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` - - CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` - CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` - CustomAzureTemplatesDirectory string `json:"custom-azure-templates-directory"` - - TemplateVersion string `json:"nuclei-templates-version,omitempty"` - NucleiVersion string `json:"nuclei-version,omitempty"` - NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` - - NucleiLatestVersion string `json:"nuclei-latest-version"` - NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"` -} - -// nucleiConfigFilename is the filename of nuclei configuration file. -const nucleiConfigFilename = ".templates-config.json" - -// Version is the current version of nuclei -const Version = `2.9.2-dev` - -var customConfigDirectory string - -func SetCustomConfigDirectory(dir string) { - customConfigDirectory = dir - if !fileutil.FolderExists(dir) { - _ = fileutil.CreateFolder(dir) - } -} -func getConfigDetails() (string, error) { - configDir, err := GetConfigDir() - if err != nil { - return "", errors.Wrap(err, "could not get home directory") - } - _ = os.MkdirAll(configDir, 0755) - templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) - return templatesConfigFile, nil -} - -// GetConfigDir returns the nuclei configuration directory -func GetConfigDir() (string, error) { - var ( - home string - err error - ) - if customConfigDirectory != "" { - home = customConfigDirectory - return home, nil - } - home, err = homedir.Dir() - if err != nil { - return "", err - } - return filepath.Join(home, ".config", "nuclei"), nil -} - -// ReadConfiguration reads the nuclei configuration file from disk. -func ReadConfiguration() (*Config, error) { - templatesConfigFile, err := getConfigDetails() - if err != nil { - return nil, err - } - file, err := os.Open(templatesConfigFile) - if err != nil { - return nil, err - } - defer file.Close() - - config := &Config{} - if err := jsoniter.NewDecoder(file).Decode(config); err != nil { - return nil, err - } - return config, nil -} - -// WriteConfiguration writes the updated nuclei configuration to disk -func WriteConfiguration(config *Config) error { - config.NucleiVersion = Version - - templatesConfigFile, err := getConfigDetails() - if err != nil { - return err - } - file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer file.Close() - - err = jsoniter.NewEncoder(file).Encode(config) - if err != nil { - return err - } - return nil -} - -const nucleiIgnoreFile = ".nuclei-ignore" - -// IgnoreFile is an internal nuclei template blocking configuration file -type IgnoreFile struct { - Tags []string `yaml:"tags"` - Files []string `yaml:"files"` -} - -// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths -func ReadIgnoreFile() IgnoreFile { - file, err := os.Open(GetIgnoreFilePath()) - if err != nil { - gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) - return IgnoreFile{} - } - defer file.Close() - - ignore := IgnoreFile{} - if err := yaml.NewDecoder(file).Decode(&ignore); err != nil { - gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) - return IgnoreFile{} - } - return ignore -} - -var ( - // customIgnoreFilePath contains a custom path for the ignore file - customIgnoreFilePath string - // ErrCustomIgnoreFilePathNotExist is raised when the ignore file doesn't exist in the custom path - ErrCustomIgnoreFilePathNotExist = errors.New("Ignore file doesn't exist in custom path") - // ErrCustomFolderNotExist is raised when the custom ignore folder doesn't exist - ErrCustomFolderNotExist = errors.New("The custom ignore path doesn't exist") -) - -// OverrideIgnoreFilePath with a custom existing folder -func OverrideIgnoreFilePath(customPath string) error { - // custom path does not exist - if !fileutil.FolderExists(customPath) { - return ErrCustomFolderNotExist - } - // ignore file within the custom path does not exist - if !fileutil.FileExists(filepath.Join(customPath, nucleiIgnoreFile)) { - return ErrCustomIgnoreFilePathNotExist - } - customIgnoreFilePath = customPath - return nil -} - -// GetIgnoreFilePath returns the ignore file path for the runner -func GetIgnoreFilePath() string { - var defIgnoreFilePath string - - if customIgnoreFilePath != "" { - defIgnoreFilePath = filepath.Join(customIgnoreFilePath, nucleiIgnoreFile) - return defIgnoreFilePath - } - - configDir, err := GetConfigDir() - if err == nil { - _ = os.MkdirAll(configDir, 0755) - - defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile) - return defIgnoreFilePath - } - cwd, err := os.Getwd() - if err != nil { - return defIgnoreFilePath - } - cwdIgnoreFilePath := filepath.Join(cwd, nucleiIgnoreFile) - return cwdIgnoreFilePath -} diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go new file mode 100644 index 0000000000..86cdf4e714 --- /dev/null +++ b/v2/pkg/catalog/config/constants.go @@ -0,0 +1,48 @@ +package config + +import ( + "strings" + + "github.com/Masterminds/semver/v3" +) + +const ( + TemplateConfigFileName = ".templates-config.json" + NucleiTemplatesDirName = "nuclei-templates" + CustomS3TemplatesDirName = "s3" + CustomGithubTemplatesDirName = "github" + OfficialNucleiTeamplatesRepoName = "nuclei-templates" + NucleiIgnoreFileName = ".nuclei-ignore" + NucleiTemplatesCheckSumFileName = ".checksum" + NewTemplateAdditionsFileName = ".new-additions" + CLIConifgFileName = "config.yaml" + ReportingConfigFilename = "reporting-config.yaml" + // Version is the current version of nuclei + Version = `v2.9.2-dev` +) + +// IsOutdatedVersion compares two versions and returns true +// if current version is outdated +func IsOutdatedVersion(current, latest string) bool { + if latest == "" { + // if pdtm api call failed it's assumed that current version is outdated + // and it will be confirmed while updating from github + // this fixes `version string empty` errors + return true + } + current = trimDevIfExists(current) + currentVer, _ := semver.NewVersion(current) + newVer, _ := semver.NewVersion(latest) + if currentVer == nil || newVer == nil { + // fallback to naive comparison + return current == latest + } + return newVer.GreaterThan(currentVer) +} + +func trimDevIfExists(version string) string { + if strings.HasSuffix(version, "-dev") { + return strings.TrimSuffix(version, "-dev") + } + return version +} diff --git a/v2/pkg/catalog/config/ignorefile.go b/v2/pkg/catalog/config/ignorefile.go new file mode 100644 index 0000000000..b8a03544f9 --- /dev/null +++ b/v2/pkg/catalog/config/ignorefile.go @@ -0,0 +1,31 @@ +package config + +import ( + "os" + + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v2" +) + +// IgnoreFile is an internal nuclei template blocking configuration file +type IgnoreFile struct { + Tags []string `yaml:"tags"` + Files []string `yaml:"files"` +} + +// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths +func ReadIgnoreFile() IgnoreFile { + file, err := os.Open(DefaultConfig.GetIgnoreFilePath()) + if err != nil { + gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) + return IgnoreFile{} + } + defer file.Close() + + ignore := IgnoreFile{} + if err := yaml.NewDecoder(file).Decode(&ignore); err != nil { + gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) + return IgnoreFile{} + } + return ignore +} diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go new file mode 100644 index 0000000000..ad5f0ea17e --- /dev/null +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -0,0 +1,302 @@ +package config + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/projectdiscovery/gologger" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" +) + +// DefaultConfig is the default nuclei configuration +// all config values and default are centralized here +var DefaultConfig *Config + +type Config struct { + TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` + + CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` + CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` + + TemplateVersion string `json:"nuclei-templates-version,omitempty"` + NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` + + // Latestxxx are not meant to be used directly and is used as + // local cache of nuclei version check endpoint + // these fields are only update during nuclei version check + // TODO: move these fields to a separate unexported struct as they are not meant to be used directly + LatestNucleiVersion string `json:"nuclei-latest-version"` + LatestNucleiTemplatesVersion string `json:"nuclei-templates-latest-version"` + LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` + + // internal / unexported fields + disableUpdates bool `json:"-"` // disable updates both version check and template updates + homeDir string `json:"-"` // User Home Directory + configDir string `json:"-"` // Nuclei Global Config Directory +} + +// WriteVersionCheckData writes version check data to config file +func (c *Config) WriteVersionCheckData(ignorehash, nucleiVersion, templatesVersion string) error { + updated := false + if ignorehash != "" && c.LatestNucleiIgnoreHash != ignorehash { + c.LatestNucleiIgnoreHash = ignorehash + updated = true + } + if nucleiVersion != "" && c.LatestNucleiVersion != nucleiVersion { + c.LatestNucleiVersion = nucleiVersion + updated = true + } + if templatesVersion != "" && c.LatestNucleiTemplatesVersion != templatesVersion { + c.LatestNucleiTemplatesVersion = templatesVersion + updated = true + } + // write config to disk if any of the fields are updated + if updated { + return c.WriteTemplatesConfig() + } + return nil +} + +// DisableUpdateCheck disables update check and template updates +func (c *Config) DisableUpdateCheck() { + c.disableUpdates = true +} + +// CanCheckForUpdates returns true if update check is enabled +func (c *Config) CanCheckForUpdates() bool { + return !c.disableUpdates +} + +// NeedsTemplateUpdate returns true if template installation/update is required +func (c *Config) NeedsTemplateUpdate() bool { + return !c.disableUpdates && (c.TemplateVersion == "" || IsOutdatedVersion(c.TemplateVersion, c.LatestNucleiTemplatesVersion) || !fileutil.FolderExists(c.TemplatesDirectory)) +} + +// NeedsIngoreFileUpdate returns true if Ignore file hash is different (aka ignore file is outdated) +func (c *Config) NeedsIgnoreFileUpdate() bool { + return c.NucleiIgnoreHash == "" || c.NucleiIgnoreHash != c.LatestNucleiIgnoreHash +} + +// UpdateNucleiIgnoreHash updates the nuclei ignore hash in config +func (c *Config) UpdateNucleiIgnoreHash() error { + // calculate hash of ignore file and update config + ignoreFilePath := c.GetIgnoreFilePath() + if fileutil.FileExists(ignoreFilePath) { + bin, err := os.ReadFile(ignoreFilePath) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not read nuclei ignore file") + } + c.NucleiIgnoreHash = fmt.Sprintf("%x", md5.Sum(bin)) + // write config to disk + return c.WriteTemplatesConfig() + } + return errorutil.NewWithTag("config", "ignore file not found: could not update nuclei ignore hash") +} + +// GetConfigDir returns the nuclei configuration directory +func (c *Config) GetConfigDir() string { + return c.configDir +} + +// GetReportingConfigFilePath returns the nuclei reporting config file path +func (c *Config) GetReportingConfigFilePath() string { + return filepath.Join(c.configDir, ReportingConfigFilename) +} + +// GetIgnoreFilePath returns the nuclei ignore file path +func (c *Config) GetIgnoreFilePath() string { + return filepath.Join(c.configDir, NucleiIgnoreFileName) +} + +// GetTemplatesConfigFilePath returns checksum file path of nuclei templates +func (c *Config) GetChecksumFilePath() string { + return filepath.Join(c.TemplatesDirectory, NucleiTemplatesCheckSumFileName) +} + +// GetCLIOptsConfigFilePath returns the nuclei cli config file path +func (c *Config) GetFlagsConfigFilePath() string { + return filepath.Join(c.configDir, CLIConifgFileName) +} + +// GetNewAdditions returns new template additions in current template release +// if .new-additions file is not present empty slice is returned +func (c *Config) GetNewAdditions() []string { + arr := []string{} + newAdditionsPath := filepath.Join(c.TemplatesDirectory, NewTemplateAdditionsFileName) + if !fileutil.FileExists(newAdditionsPath) { + return arr + } + bin, err := os.ReadFile(newAdditionsPath) + if err != nil { + return arr + } + for _, v := range strings.Fields(string(bin)) { + if IsTemplate(v) { + arr = append(arr, v) + } + } + return arr +} + +// SetConfigDir sets the nuclei configuration directory +// and appropriate changes are made to the config +func (c *Config) SetConfigDir(dir string) { + c.configDir = dir + if err := c.createConfigDirIfNotExists(); err != nil { + gologger.Fatal().Msgf("Could not create nuclei config directory at %s: %s", c.configDir, err) + } + + // if folder already exists read config or create new + if err := c.ReadTemplatesConfig(); err != nil { + // create new config + applyDefaultConfig() + if err2 := c.WriteTemplatesConfig(); err2 != nil { + gologger.Fatal().Msgf("Could not create nuclei config file at %s: %s", c.getTemplatesConfigFilePath(), err2) + } + } + + // while other config files are optional, ignore file is mandatory + // since it is used to ignore templates with weak matchers + c.copyIgnoreFile() +} + +// SetTemplatesDir sets the new nuclei templates directory +func (c *Config) SetTemplatesDir(dirPath string) { + if dirPath != "" && !filepath.IsAbs(dirPath) { + cwd, _ := os.Getwd() + dirPath = filepath.Join(cwd, dirPath) + } + c.TemplatesDirectory = dirPath + // Update the custom templates directory + c.CustomGithubTemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) + c.CustomS3TemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) +} + +// SetTemplatesVersion sets the new nuclei templates version +func (c *Config) SetTemplatesVersion(version string) error { + c.TemplateVersion = version + // write config to disk + if err := c.WriteTemplatesConfig(); err != nil { + return errorutil.NewWithErr(err).Msgf("could not write nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + return nil +} + +// ReadTemplatesConfig reads the nuclei templates config file +func (c *Config) ReadTemplatesConfig() error { + if !fileutil.FileExists(c.getTemplatesConfigFilePath()) { + return errorutil.NewWithTag("config", "nuclei config file at %s does not exist", c.getTemplatesConfigFilePath()) + } + var cfg *Config + bin, err := os.ReadFile(c.getTemplatesConfigFilePath()) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not read nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + if err := json.Unmarshal(bin, &cfg); err != nil { + return errorutil.NewWithErr(err).Msgf("could not unmarshal nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + // apply config + c.CustomGithubTemplatesDirectory = cfg.CustomGithubTemplatesDirectory + c.CustomS3TemplatesDirectory = cfg.CustomS3TemplatesDirectory + c.TemplatesDirectory = cfg.TemplatesDirectory + c.TemplateVersion = cfg.TemplateVersion + c.NucleiIgnoreHash = cfg.NucleiIgnoreHash + c.LatestNucleiIgnoreHash = cfg.LatestNucleiIgnoreHash + c.LatestNucleiTemplatesVersion = cfg.LatestNucleiTemplatesVersion + return nil +} + +// WriteTemplatesConfig writes the nuclei templates config file +func (c *Config) WriteTemplatesConfig() error { + // check if config folder exists if not create one + if err := c.createConfigDirIfNotExists(); err != nil { + return err + } + bin, err := json.Marshal(c) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to marshal nuclei config") + } + if err = os.WriteFile(c.getTemplatesConfigFilePath(), bin, 0600); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + return nil +} + +// getTemplatesConfigFilePath returns configDir/.templates-config.json file path +func (c *Config) getTemplatesConfigFilePath() string { + return filepath.Join(c.configDir, TemplateConfigFileName) +} + +// createConfigDirIfNotExists creates the nuclei config directory if not exists +func (c *Config) createConfigDirIfNotExists() error { + if !fileutil.FolderExists(c.configDir) { + if err := fileutil.CreateFolder(c.configDir); err != nil { + return errorutil.NewWithErr(err).Msgf("could not create nuclei config directory at %s", c.configDir) + } + } + return nil +} + +// copyIgnoreFile copies the nuclei ignore file default config directory +// to the current config directory +func (c *Config) copyIgnoreFile() { + if err := c.createConfigDirIfNotExists(); err != nil { + gologger.Error().Msgf("Could not create nuclei config directory at %s: %s", c.configDir, err) + return + } + ignoreFilePath := c.GetIgnoreFilePath() + if !fileutil.FileExists(ignoreFilePath) { + // copy ignore file + if err := fileutil.CopyFile(filepath.Join(getDefaultConfigDir(), NucleiIgnoreFileName), ignoreFilePath); err != nil { + gologger.Error().Msgf("Could not copy nuclei ignore file at %s: %s", ignoreFilePath, err) + } + } +} + +func init() { + ConfigDir := getDefaultConfigDir() + if !fileutil.FolderExists(ConfigDir) { + if err := fileutil.CreateFolder(ConfigDir); err != nil { + gologger.Error().Msgf("failed to create config directory at %v got: %s", ConfigDir, err) + } + } + DefaultConfig = &Config{ + homeDir: folderutil.HomeDirOrDefault(""), + configDir: ConfigDir, + } + // try to read config from file + if err := DefaultConfig.ReadTemplatesConfig(); err != nil { + gologger.Verbose().Msgf("config file not found, creating new config file at %s", DefaultConfig.getTemplatesConfigFilePath()) + applyDefaultConfig() + // write config to file + if err := DefaultConfig.WriteTemplatesConfig(); err != nil { + gologger.Error().Msgf("failed to write config file at %s got: %s", DefaultConfig.getTemplatesConfigFilePath(), err) + } + } +} + +func getDefaultConfigDir() string { + // Review Needed: Earlier a dependency was used to locate home dir + // i.e "github.com/mitchellh/go-homedir" not sure if it is needed + // Even if such case exists it should be abstracted via below function call in utils/folder + homedir := folderutil.HomeDirOrDefault("") + // TBD: we should probably stick to specification and use config directories provided by distro + // instead of manually creating one since $HOME/.config/ is config directory of Linux desktops + // Ref: https://pkg.go.dev/os#UserConfigDir + // some distros like NixOS or others have totally different config directories this causes issues for us (since we are not using os.UserConfigDir) + userCfgDir := filepath.Join(homedir, ".config") + return filepath.Join(userCfgDir, "nuclei") +} + +// Add Default Config adds default when .templates-config.json file is not present +func applyDefaultConfig() { + DefaultConfig.TemplatesDirectory = filepath.Join(DefaultConfig.homeDir, NucleiTemplatesDirName) + DefaultConfig.CustomGithubTemplatesDirectory = filepath.Join(DefaultConfig.TemplatesDirectory, CustomGithubTemplatesDirName) + DefaultConfig.CustomS3TemplatesDirectory = filepath.Join(DefaultConfig.TemplatesDirectory, CustomS3TemplatesDirName) +} diff --git a/v2/pkg/catalog/config/template.go b/v2/pkg/catalog/config/template.go index b36bd777bf..806a67f014 100644 --- a/v2/pkg/catalog/config/template.go +++ b/v2/pkg/catalog/config/template.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" + stringsutil "github.com/projectdiscovery/utils/strings" ) // TemplateFormat @@ -33,3 +34,9 @@ func GetTemplateFormatFromExt(filePath string) TemplateFormat { func GetSupportTemplateFileExtensions() []string { return []string{extensions.YAML, extensions.JSON} } + +// isTemplate is a callback function used by goflags to decide if given file should be read +// if it is not a nuclei-template file only then file is read +func IsTemplate(filename string) bool { + return stringsutil.EqualFoldAny(filepath.Ext(filename), GetSupportTemplateFileExtensions()...) +} diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 9b963d3037..773f32492a 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -7,7 +7,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + cfg "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" @@ -41,9 +41,8 @@ type Config struct { ExcludeIds []string IncludeConditions []string - Catalog catalog.Catalog - ExecutorOptions protocols.ExecuterOptions - TemplatesDirectory string + Catalog catalog.Catalog + ExecutorOptions protocols.ExecuterOptions } // Store is a storage for loaded nuclei templates @@ -65,7 +64,7 @@ type Store struct { } // NewConfig returns a new loader config -func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { +func NewConfig(options *types.Options, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { loaderConfig := Config{ Templates: options.Templates, Workflows: options.Workflows, @@ -82,7 +81,6 @@ func NewConfig(options *types.Options, templateConfig *config.Config, catalog ca IncludeTags: options.IncludeTags, IncludeIds: options.IncludeIds, ExcludeIds: options.ExcludeIds, - TemplatesDirectory: templateConfig.TemplatesDirectory, Protocols: options.Protocols, ExcludeProtocols: options.ExcludeProtocols, IncludeConditions: options.IncludeConditions, @@ -142,7 +140,7 @@ func New(config *Config) (*Store, error) { } // Handle a case with no templates or workflows, where we use base directory if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided { - store.finalTemplates = []string{config.TemplatesDirectory} + store.finalTemplates = []string{cfg.DefaultConfig.TemplatesDirectory} } return store, nil } diff --git a/v2/pkg/catalog/loader/loader_test.go b/v2/pkg/catalog/loader/loader_test.go index 69d1f7d647..9c3f4f561c 100644 --- a/v2/pkg/catalog/loader/loader_test.go +++ b/v2/pkg/catalog/loader/loader_test.go @@ -4,6 +4,7 @@ import ( "reflect" "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/stretchr/testify/require" ) @@ -19,28 +20,26 @@ func TestLoadTemplates(t *testing.T) { require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates") templatesDirectory := "/test" + config.DefaultConfig.TemplatesDirectory = templatesDirectory t.Run("blank", func(t *testing.T) { store, err := New(&Config{ - TemplatesDirectory: templatesDirectory, - Catalog: catalog, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) t.Run("only-tags", func(t *testing.T) { store, err := New(&Config{ - Tags: []string{"cves"}, - TemplatesDirectory: templatesDirectory, - Catalog: catalog, + Tags: []string{"cves"}, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) t.Run("tags-with-path", func(t *testing.T) { store, err := New(&Config{ - Tags: []string{"cves"}, - TemplatesDirectory: templatesDirectory, - Catalog: catalog, + Tags: []string{"cves"}, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index d450b34d12..a2fc123b82 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -2,6 +2,7 @@ package parsers import ( "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -40,7 +41,7 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) } func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string { - includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory}) + includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{config.DefaultConfig.TemplatesDirectory}) for template, err := range errs { gologger.Error().Msgf("Could not find template '%s': %s", template, err) } diff --git a/v2/pkg/protocols/common/automaticscan/automaticscan.go b/v2/pkg/protocols/common/automaticscan/automaticscan.go index 10bd7fe8db..780314edcb 100644 --- a/v2/pkg/protocols/common/automaticscan/automaticscan.go +++ b/v2/pkg/protocols/common/automaticscan/automaticscan.go @@ -57,7 +57,7 @@ func New(opts Options) (*Service, error) { } var mappingData map[string]string - config, err := config.ReadConfiguration() + config := config.DefaultConfig if err == nil { mappingFile := filepath.Join(config.TemplatesDirectory, mappingFilename) if file, err := os.Open(mappingFile); err == nil { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 58c2419b88..9cc8a97612 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" ) // PayloadGenerator is the generator struct for generating payloads @@ -16,7 +17,7 @@ type PayloadGenerator struct { } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath, templateDirectory string, sandbox bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, sandbox bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -42,7 +43,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath, t return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, templateDirectory, sandbox) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, sandbox) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 00a1a6212d..7187808904 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -12,7 +12,7 @@ func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -32,7 +32,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -54,7 +54,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 4f6241e4f5..7056305ea6 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -98,7 +98,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { if len(request.Payloads) > 0 { var err error - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.TemplatesDirectory, options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.Sandbox, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index a6a15b898e..7cfe74aab8 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -350,7 +350,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.TemplatesDirectory, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index b30a5cd110..89447b9936 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -34,7 +34,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) @@ -58,7 +58,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 0ac06248dc..0d8995b5ac 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.TemplatesDirectory, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 324194043f..bd5b655c41 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -105,7 +105,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.TemplatesDirectory, request.options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 23197ad1e9..7ecd4e4216 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -2,8 +2,8 @@ package reporting import ( "os" - "path/filepath" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" json_exporter "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonl" @@ -12,7 +12,6 @@ import ( "errors" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -177,11 +176,7 @@ func New(options *Options, db string) (Client, error) { // CreateConfigIfNotExists creates report-config if it doesn't exists func CreateConfigIfNotExists() error { - config, err := config.GetConfigDir() - if err != nil { - return errorutil.NewWithErr(err).Msgf("could not get config directory") - } - reportingConfig := filepath.Join(config, "report-config.yaml") + reportingConfig := config.DefaultConfig.GetReportingConfigFilePath() if fileutil.FileExists(reportingConfig) { return nil diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index a2ed7d6153..3b5fdad798 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -310,7 +310,7 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th builder.WriteString(event.CURLCommand) builder.WriteString("\n{code}") } - builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei v%s](https://github.com/projectdiscovery/nuclei)", config.Version)) + builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() return data } diff --git a/v2/pkg/templates/compile_test.go b/v2/pkg/templates/compile_test.go index 91d3720e6c..575d039332 100644 --- a/v2/pkg/templates/compile_test.go +++ b/v2/pkg/templates/compile_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -44,7 +45,7 @@ func setup() { ProjectFile: nil, IssuesClient: nil, Browser: nil, - Catalog: disk.NewCatalog(options.TemplatesDirectory), + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), } workflowLoader, err := parsers.NewLoader(&executerOpts) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index e11a439622..5d08d06d40 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -9,6 +9,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -31,14 +32,12 @@ var DefaultOptions = &types.Options{ DebugRequests: false, DebugResponse: false, Silent: false, - Version: false, Verbose: false, NoColor: true, UpdateTemplates: false, JSONL: false, JSONRequests: false, EnableProgressBar: false, - TemplatesVersion: false, TemplateList: false, Stdin: false, StopAtFirstMatch: false, @@ -56,7 +55,6 @@ var DefaultOptions = &types.Options{ TargetsFilePath: "", Output: "", Proxy: []string{}, - TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, ExcludedTemplates: []string{}, @@ -90,7 +88,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco ProjectFile: nil, IssuesClient: nil, Browser: nil, - Catalog: disk.NewCatalog(options.TemplatesDirectory), + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), } return executerOpts diff --git a/v2/pkg/types/resume.go b/v2/pkg/types/resume.go index 198631ad73..b65a930145 100644 --- a/v2/pkg/types/resume.go +++ b/v2/pkg/types/resume.go @@ -14,10 +14,7 @@ import ( const DefaultResumeFileName = "resume-%s.cfg" func DefaultResumeFilePath() string { - configDir, err := config.GetConfigDir() - if err != nil { - return fmt.Sprintf(DefaultResumeFileName, xid.New().String()) - } + configDir := config.DefaultConfig.GetConfigDir() resumeFile := filepath.Join(configDir, fmt.Sprintf(DefaultResumeFileName, xid.New().String())) return resumeFile } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 10ea6d8972..0cae038945 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -76,7 +76,7 @@ type Options struct { // List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) Proxy goflags.StringSlice // TemplatesDirectory is the directory to use for storing templates - TemplatesDirectory string + NewTemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests TraceLogFile string // ErrorLogFile specifies a file to write with the errors of all requests @@ -215,8 +215,6 @@ type Options struct { AutomaticScan bool // Silent suppresses any extra text and only writes found URLs on screen. Silent bool - // Version specifies if we should just show version and exit - Version bool // Validate validates the templates passed to nuclei. Validate bool // NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs). @@ -228,7 +226,7 @@ type Options struct { ShowVarDump bool // No-Color disables the colored output. NoColor bool - // UpdateTemplates updates the templates installed at startup + // UpdateTemplates updates the templates installed at startup (also used by cloud to update datasources) UpdateTemplates bool // JSON writes json line output to files JSONL bool @@ -242,8 +240,6 @@ type Options struct { Cloud bool // EnableProgressBar enables progress bar EnableProgressBar bool - // TemplatesVersion shows the templates installed version - TemplatesVersion bool // TemplateDisplay displays the template contents TemplateDisplay bool // TemplateList lists available templates @@ -268,10 +264,6 @@ type Options struct { NewTemplatesWithVersion goflags.StringSlice // NoInteractsh disables use of interactsh server for interaction polling NoInteractsh bool - // UpdateNuclei checks for an update for the nuclei engine - UpdateNuclei bool - // NoUpdateTemplates disables checking for nuclei templates updates - NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool // MatcherStatus displays optional status for the failed matches as well @@ -330,8 +322,6 @@ type Options struct { UncoverLimit int // Uncover search delay UncoverDelay int - // ConfigPath contains the config path (used by healthcheck) - ConfigPath string // ScanAllIPs associated to a dns record ScanAllIPs bool // IPVersion to scan (4,6) diff --git a/v2/pkg/utils/template_path.go b/v2/pkg/utils/template_path.go index 7fac7e4a44..0e54d9904c 100644 --- a/v2/pkg/utils/template_path.go +++ b/v2/pkg/utils/template_path.go @@ -1,10 +1,8 @@ package utils import ( - "path/filepath" "strings" - "github.com/mitchellh/go-homedir" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" ) @@ -13,16 +11,11 @@ const ( TemplatesRepoURL = "https://github.com/projectdiscovery/nuclei-templates/blob/main/" ) -var configData *config.Config - -func init() { - configData, _ = config.ReadConfiguration() -} - // TemplatePathURL returns the Path and URL for the provided template func TemplatePathURL(fullPath string) (string, string) { var templateDirectory string - if configData != nil && configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { + configData := config.DefaultConfig + if configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { templateDirectory = configData.TemplatesDirectory } else { return "", "" @@ -32,12 +25,3 @@ func TemplatePathURL(fullPath string) (string, string) { templateURL := TemplatesRepoURL + finalPath return finalPath, templateURL } - -// GetDefaultTemplatePath on default settings -func GetDefaultTemplatePath() (string, error) { - home, err := homedir.Dir() - if err != nil { - return "", err - } - return filepath.Join(home, "nuclei-templates"), nil -} From 978d0bcc23bec7a15d02107aeef074107148a51c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 19 Apr 2023 19:30:15 +0200 Subject: [PATCH 31/34] Replacing goos with osutils (#3571) * Replacing goos with osutils * pleasing his majesty the linter --- v2/cmd/nuclei/main.go | 5 +++-- v2/pkg/protocols/common/generators/load_test.go | 4 ++-- v2/pkg/protocols/headless/engine/engine.go | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 4f7c4316ad..5c3e4913e1 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -25,6 +25,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/monitor" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + osutils "github.com/projectdiscovery/utils/os" ) var ( @@ -357,9 +358,9 @@ on extensive configurability, massive extensibility and ease of use.`) return flagSet } -// isColorNotAvailable returns true if ascii collored output is not available. +// isColorNotAvailable returns true if ascii colored output is not available. func isColorNotAvailable() bool { - return runtime.GOOS == "windows" + return osutils.IsWindows() } // cleanupOldResumeFiles cleans up resume files older than 10 days. diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index 9aeaf02269..cecdde9b5a 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -3,10 +3,10 @@ package generators import ( "os" "path/filepath" - "runtime" "testing" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + osutils "github.com/projectdiscovery/utils/os" "github.com/stretchr/testify/require" ) @@ -37,7 +37,7 @@ func TestLoadPayloads(t *testing.T) { require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) t.Run("no-sandbox-unix", func(t *testing.T) { - if runtime.GOOS == "windows" { + if osutils.IsWindows() { return } _, err := generator.loadPayloads(map[string]interface{}{ diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 41c6f5c706..a547f2a774 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "os" - "runtime" "strings" "github.com/go-rod/rod" @@ -14,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" fileutil "github.com/projectdiscovery/utils/file" + osutils "github.com/projectdiscovery/utils/os" reflectutil "github.com/projectdiscovery/utils/reflect" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -123,7 +123,7 @@ func New(options *types.Options) (*Browser, error) { func MustDisableSandbox() bool { // linux with root user needs "--no-sandbox" option // https://github.com/chromium/chromium/blob/c4d3c31083a2e1481253ff2d24298a1dfe19c754/chrome/test/chromedriver/client/chromedriver.py#L209 - return runtime.GOOS == "linux" && os.Geteuid() == 0 + return osutils.IsWindows() && os.Geteuid() == 0 } // SetUserAgent sets custom user agent to the browser From b211d6fa441be5bd66f486c83d7d6371a8867c3c Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Wed, 19 Apr 2023 16:52:34 -0400 Subject: [PATCH 32/34] Duplicate error message, variable collision fix, method doc references (#3568) --- v2/pkg/external/customtemplates/s3.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/external/customtemplates/s3.go b/v2/pkg/external/customtemplates/s3.go index cd1683b0aa..674fceb0d5 100644 --- a/v2/pkg/external/customtemplates/s3.go +++ b/v2/pkg/external/customtemplates/s3.go @@ -21,11 +21,11 @@ type customTemplateS3Bucket struct { Location string } -// download custom templates from s3 bucket +// Download retrieves all custom templates from s3 bucket func (bk *customTemplateS3Bucket) Download(location string, ctx context.Context) { downloadPath := filepath.Join(location, CustomS3TemplateDirectory, bk.bucketName) - manager := manager.NewDownloader(bk.s3Client) + s3Manager := manager.NewDownloader(bk.s3Client) paginator := s3.NewListObjectsV2Paginator(bk.s3Client, &s3.ListObjectsV2Input{ Bucket: &bk.bucketName, Prefix: &bk.prefix, @@ -38,16 +38,16 @@ func (bk *customTemplateS3Bucket) Download(location string, ctx context.Context) return } for _, obj := range page.Contents { - if err := downloadToFile(manager, downloadPath, bk.bucketName, aws.ToString(obj.Key)); err != nil { + if err := downloadToFile(s3Manager, downloadPath, bk.bucketName, aws.ToString(obj.Key)); err != nil { gologger.Error().Msgf("error downloading s3 bucket %s %s", bk.bucketName, err) return } } } - gologger.Info().Msgf("AWS bucket %s successfully cloned successfully at %s", bk.bucketName, downloadPath) + gologger.Info().Msgf("AWS bucket %s was cloned successfully at %s", bk.bucketName, downloadPath) } -// download custom templates from s3 bucket +// Update download custom templates from s3 bucket func (bk *customTemplateS3Bucket) Update(location string, ctx context.Context) { bk.Download(location, ctx) } From dcb003211ca51fd0fcef30db66e92fd79045cb14 Mon Sep 17 00:00:00 2001 From: Keith Chason Date: Wed, 19 Apr 2023 17:42:52 -0400 Subject: [PATCH 33/34] Gitlab Custom Templates (#3570) * Configuration options for GitLab template pulls * GitLab client creation * GitLab hooks and property renames * Fix filesystem writing and update environment variables * Fix type error in formatted error message * Migrate directory config to new nucleiconfig file * refactor + add custom templates to tm * typo fix + only show installed ct with -tv * add default gitlab url if not given * fix template valid failure --------- Co-authored-by: Tarun Koyalwar --- v2/cmd/nuclei/main.go | 11 +- v2/internal/installer/template.go | 16 +- v2/internal/runner/options.go | 48 +++++- v2/internal/runner/runner.go | 25 ++- v2/pkg/catalog/config/constants.go | 8 +- v2/pkg/catalog/config/nucleiconfig.go | 24 ++- v2/pkg/external/customtemplates/azure_blob.go | 42 ++++- v2/pkg/external/customtemplates/github.go | 42 ++++- .../external/customtemplates/github_test.go | 11 +- v2/pkg/external/customtemplates/gitlab.go | 144 ++++++++++++++++++ v2/pkg/external/customtemplates/s3.go | 38 ++++- .../customtemplates/templates_provider.go | 125 ++++++++------- v2/pkg/types/types.go | 18 ++- 13 files changed, 442 insertions(+), 110 deletions(-) create mode 100644 v2/pkg/external/customtemplates/gitlab.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 5c3e4913e1..20b86dbe68 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -417,12 +417,19 @@ func printVersion() { func printTemplateVersion() { cfg := config.DefaultConfig gologger.Info().Msgf("Public nuclei-templates version: %s (%s)\n", cfg.TemplateVersion, cfg.TemplatesDirectory) - if cfg.CustomS3TemplatesDirectory != "" { + + if fileutil.FolderExists(cfg.CustomS3TemplatesDirectory) { gologger.Info().Msgf("Custom S3 templates location: %s\n", cfg.CustomS3TemplatesDirectory) } - if cfg.CustomGithubTemplatesDirectory != "" { + if fileutil.FolderExists(cfg.CustomGithubTemplatesDirectory) { gologger.Info().Msgf("Custom Github templates location: %s ", cfg.CustomGithubTemplatesDirectory) } + if fileutil.FolderExists(cfg.CustomGitLabTemplatesDirectory) { + gologger.Info().Msgf("Custom Gitlab templates location: %s ", cfg.CustomGitLabTemplatesDirectory) + } + if fileutil.FolderExists(cfg.CustomAzureTemplatesDirectory) { + gologger.Info().Msgf("Custom Azure templates location: %s ", cfg.CustomAzureTemplatesDirectory) + } os.Exit(0) } diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go index 8c0f671a0e..9a0c849231 100644 --- a/v2/internal/installer/template.go +++ b/v2/internal/installer/template.go @@ -2,6 +2,7 @@ package installer import ( "bytes" + "context" "crypto/md5" "fmt" "io" @@ -14,6 +15,7 @@ import ( "github.com/olekukonko/tablewriter" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/external/customtemplates" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" @@ -54,7 +56,9 @@ func (t *templateUpdateResults) String() string { // TemplateManager is a manager for templates. // It downloads / updates / installs templates. -type TemplateManager struct{} +type TemplateManager struct { + CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates +} // FreshInstallIfNotExists installs templates if they are not already installed // if templates directory already exists, it does nothing @@ -63,7 +67,13 @@ func (t *TemplateManager) FreshInstallIfNotExists() error { return nil } gologger.Info().Msgf("nuclei-templates are not installed, installing...") - return t.installTemplatesAt(config.DefaultConfig.TemplatesDirectory) + if err := t.installTemplatesAt(config.DefaultConfig.TemplatesDirectory); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", config.DefaultConfig.TemplatesDirectory) + } + if t.CustomTemplates != nil { + t.CustomTemplates.Download(context.TODO()) + } + return nil } // UpdateIfOutdated updates templates if they are outdated @@ -310,7 +320,7 @@ func (t *TemplateManager) calculateChecksumMap(dir string) (map[string]string, e return err } // skip checksums of custom templates i.e github and s3 - if stringsutil.HasPrefixAny(path, config.DefaultConfig.CustomGithubTemplatesDirectory, config.DefaultConfig.CustomS3TemplatesDirectory) { + if stringsutil.HasPrefixAny(path, config.DefaultConfig.GetAllCustomTemplateDirs()...) { return nil } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 7a89f7de35..851f7edf8d 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "github.com/pkg/errors" @@ -129,7 +130,7 @@ func validateOptions(options *types.Options) error { } validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile}) } - // Verify aws secrets are passed if s3 template bucket passed + // Verify AWS secrets are passed if a S3 template bucket is passed if options.AwsBucketName != "" && options.UpdateTemplates { missing := validateMissingS3Options(options) if missing != nil { @@ -145,6 +146,14 @@ func validateOptions(options *types.Options) error { } } + // Verify that all GitLab options are provided if the GitLab server or token is provided + if options.GitLabToken != "" && options.UpdateTemplates { + missing := validateMissingGitLabOptions(options) + if missing != nil { + return fmt.Errorf("gitlab server details are missing. Please provide %s", strings.Join(missing, ",")) + } + } + // verify that a valid ip version type was selected (4, 6) if len(options.IPVersion) == 0 { // add ipv4 as default @@ -186,6 +195,8 @@ func validateCloudOptions(options *types.Options) error { missing = validateMissingS3Options(options) case "github": missing = validateMissingGithubOptions(options) + case "gitlab": + missing = validateMissingGitLabOptions(options) case "azure": missing = validateMissingAzureOptions(options) } @@ -244,6 +255,18 @@ func validateMissingGithubOptions(options *types.Options) []string { return missing } +func validateMissingGitLabOptions(options *types.Options) []string { + var missing []string + if options.GitLabToken == "" { + missing = append(missing, "GITLAB_TOKEN") + } + if len(options.GitLabTemplateRepositoryIDs) == 0 { + missing = append(missing, "GITLAB_REPOSITORY_IDS") + } + + return missing +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -333,6 +356,29 @@ func readEnvInputVars(options *types.Options) { if repolist != "" { options.GithubTemplateRepo = append(options.GithubTemplateRepo, stringsutil.SplitAny(repolist, ",")...) } + + // GitLab options for downloading templates from a repository + options.GitLabServerURL = os.Getenv("GITLAB_SERVER_URL") + if options.GitLabServerURL == "" { + options.GitLabServerURL = "https://gitlab.com" + } + options.GitLabToken = os.Getenv("GITLAB_TOKEN") + repolist = os.Getenv("GITLAB_REPOSITORY_IDS") + // Convert the comma separated list of repository IDs to a list of integers + if repolist != "" { + for _, repoID := range stringsutil.SplitAny(repolist, ",") { + // Attempt to convert the repo ID to an integer + repoIDInt, err := strconv.Atoi(repoID) + if err != nil { + gologger.Warning().Msgf("Invalid GitLab template repository ID: %s", repoID) + continue + } + + // Add the int repository ID to the list + options.GitLabTemplateRepositoryIDs = append(options.GitLabTemplateRepositoryIDs, repoIDInt) + } + } + // AWS options for downloading templates from an S3 bucket options.AwsAccessKey = os.Getenv("AWS_ACCESS_KEY") options.AwsSecretKey = os.Getenv("AWS_SECRET_KEY") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 54039e909d..18980d3012 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -73,7 +73,6 @@ type Runner struct { hostErrors hosterrorscache.CacheInterface resumeCfg *types.ResumeCfg pprofServer *http.Server - customTemplates []customtemplates.Provider cloudClient *nucleicloud.Client cloudTargets []string } @@ -102,8 +101,16 @@ func New(options *types.Options) (*Runner, error) { gologger.Error().Msgf("nuclei version check failed got: %s\n", err) } } + + // check for custom template updates and update if available + ctm, err := customtemplates.NewCustomTemplatesManager(options) + if err != nil { + gologger.Error().Label("custom-templates").Msgf("Failed to create custom templates manager: %s\n", err) + } + // Check for template updates and update if available - tm := &installer.TemplateManager{} + // if custom templates manager is not nil, we will install custom templates if there is fresh installation + tm := &installer.TemplateManager{CustomTemplates: ctm} if err := tm.FreshInstallIfNotExists(); err != nil { gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err) } @@ -116,6 +123,18 @@ func New(options *types.Options) (*Runner, error) { gologger.Warning().Msgf("failed to update nuclei ignore file: %s\n", err) } } + + if options.UpdateTemplates { + // we automatically check for updates unless explicitly disabled + // this print statement is only to inform the user that there are no updates + if !config.DefaultConfig.NeedsTemplateUpdate() { + gologger.Info().Msgf("No new updates found for nuclei templates") + } + // manually trigger update of custom templates + if ctm != nil { + ctm.Update(context.TODO()) + } + } } if options.Validate { @@ -125,8 +144,6 @@ func New(options *types.Options) (*Runner, error) { // TODO: refactor to pass options reference globally without cycles parsers.NoStrictSyntax = options.NoStrictSyntax yaml.StrictSyntax = !options.NoStrictSyntax - // parse the runner.options.GithubTemplateRepo and store the valid repos in runner.customTemplateRepos - runner.customTemplates = customtemplates.ParseCustomTemplates(runner.options) if options.Headless { if engine.MustDisableSandbox() { diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 86cdf4e714..9e4a06e29d 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -9,8 +9,6 @@ import ( const ( TemplateConfigFileName = ".templates-config.json" NucleiTemplatesDirName = "nuclei-templates" - CustomS3TemplatesDirName = "s3" - CustomGithubTemplatesDirName = "github" OfficialNucleiTeamplatesRepoName = "nuclei-templates" NucleiIgnoreFileName = ".nuclei-ignore" NucleiTemplatesCheckSumFileName = ".checksum" @@ -19,6 +17,12 @@ const ( ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei Version = `v2.9.2-dev` + + // Directory Names of custom templates + CustomS3TemplatesDirName = "s3" + CustomGithubTemplatesDirName = "github" + CustomAzureTemplatesDirName = "azure" + CustomGitLabTemplatesDirName = "gitlab" ) // IsOutdatedVersion compares two versions and returns true diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go index ad5f0ea17e..59e0646c6b 100644 --- a/v2/pkg/catalog/config/nucleiconfig.go +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -21,8 +21,12 @@ var DefaultConfig *Config type Config struct { TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` + // customtemplates exists in templates directory with the name of custom-templates provider + // below custom paths are absolute paths to respecitive custom-templates directories CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` + CustomGitLabTemplatesDirectory string `json:"custom-gitlab-templates-directory"` + CustomAzureTemplatesDirectory string `json:"custom-azure-templates-directory"` TemplateVersion string `json:"nuclei-templates-version,omitempty"` NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` @@ -104,6 +108,11 @@ func (c *Config) GetConfigDir() string { return c.configDir } +// GetAllCustomTemplateDirs returns all custom template directories +func (c *Config) GetAllCustomTemplateDirs() []string { + return []string{c.CustomS3TemplatesDirectory, c.CustomGithubTemplatesDirectory, c.CustomGitLabTemplatesDirectory, c.CustomAzureTemplatesDirectory} +} + // GetReportingConfigFilePath returns the nuclei reporting config file path func (c *Config) GetReportingConfigFilePath() string { return filepath.Join(c.configDir, ReportingConfigFilename) @@ -175,7 +184,9 @@ func (c *Config) SetTemplatesDir(dirPath string) { c.TemplatesDirectory = dirPath // Update the custom templates directory c.CustomGithubTemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) - c.CustomS3TemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) + c.CustomS3TemplatesDirectory = filepath.Join(dirPath, CustomS3TemplatesDirName) + c.CustomGitLabTemplatesDirectory = filepath.Join(dirPath, CustomGitLabTemplatesDirName) + c.CustomAzureTemplatesDirectory = filepath.Join(dirPath, CustomAzureTemplatesDirName) } // SetTemplatesVersion sets the new nuclei templates version @@ -202,8 +213,6 @@ func (c *Config) ReadTemplatesConfig() error { return errorutil.NewWithErr(err).Msgf("could not unmarshal nuclei config file at %s", c.getTemplatesConfigFilePath()) } // apply config - c.CustomGithubTemplatesDirectory = cfg.CustomGithubTemplatesDirectory - c.CustomS3TemplatesDirectory = cfg.CustomS3TemplatesDirectory c.TemplatesDirectory = cfg.TemplatesDirectory c.TemplateVersion = cfg.TemplateVersion c.NucleiIgnoreHash = cfg.NucleiIgnoreHash @@ -279,6 +288,11 @@ func init() { gologger.Error().Msgf("failed to write config file at %s got: %s", DefaultConfig.getTemplatesConfigFilePath(), err) } } + // Loads/updates paths of custom templates + // Note: custom templates paths should not be updated in config file + // and even if it is changed we don't follow it since it is not expected behavior + // If custom templates are in default locations only then they are loaded while running nuclei + DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory) } func getDefaultConfigDir() string { @@ -297,6 +311,6 @@ func getDefaultConfigDir() string { // Add Default Config adds default when .templates-config.json file is not present func applyDefaultConfig() { DefaultConfig.TemplatesDirectory = filepath.Join(DefaultConfig.homeDir, NucleiTemplatesDirName) - DefaultConfig.CustomGithubTemplatesDirectory = filepath.Join(DefaultConfig.TemplatesDirectory, CustomGithubTemplatesDirName) - DefaultConfig.CustomS3TemplatesDirectory = filepath.Join(DefaultConfig.TemplatesDirectory, CustomS3TemplatesDirName) + // updates all necessary paths + DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory) } diff --git a/v2/pkg/external/customtemplates/azure_blob.go b/v2/pkg/external/customtemplates/azure_blob.go index 76a6ea9144..4e405caf5a 100644 --- a/v2/pkg/external/customtemplates/azure_blob.go +++ b/v2/pkg/external/customtemplates/azure_blob.go @@ -3,19 +3,47 @@ package customtemplates import ( "bytes" "context" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/projectdiscovery/gologger" "os" "path/filepath" "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" ) +var _ Provider = &customTemplateAzureBlob{} + type customTemplateAzureBlob struct { azureBlobClient *azblob.Client containerName string } +// NewAzureProviders creates a new Azure Blob Storage provider for downloading custom templates +func NewAzureProviders(options *types.Options) ([]*customTemplateAzureBlob, error) { + providers := []*customTemplateAzureBlob{} + if options.AzureContainerName != "" { + // Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage + azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("Error establishing Azure Blob client for %s", options.AzureContainerName) + } + + // Create a new Azure Blob Storage container object + azTemplateContainer := &customTemplateAzureBlob{ + azureBlobClient: azClient, + containerName: options.AzureContainerName, + } + + // Add the Azure Blob Storage container object to the list of custom templates + providers = append(providers, azTemplateContainer) + } + return providers, nil +} + func getAzureBlobClient(tenantID string, clientID string, clientSecret string, serviceURL string) (*azblob.Client, error) { // Create an Azure credential using the provided credentials credentials, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) @@ -34,12 +62,12 @@ func getAzureBlobClient(tenantID string, clientID string, clientSecret string, s return client, nil } -func (bk *customTemplateAzureBlob) Download(location string, ctx context.Context) { +func (bk *customTemplateAzureBlob) Download(ctx context.Context) { // Set an incrementer for the number of templates downloaded var templatesDownloaded = 0 // Define the local path to which the templates will be downloaded - downloadPath := filepath.Join(location, CustomAzureTemplateDirectory, bk.containerName) + downloadPath := filepath.Join(config.DefaultConfig.CustomAzureTemplatesDirectory, bk.containerName) // Get the list of all templates from the container pager := bk.azureBlobClient.NewListBlobsFlatPager(bk.containerName, &azblob.ListBlobsFlatOptions{ @@ -78,9 +106,9 @@ func (bk *customTemplateAzureBlob) Download(location string, ctx context.Context // Update updates the templates from the Azure Blob Storage container to the local filesystem. This is effectively a // wrapper of the Download function which downloads of all templates from the container and doesn't manage a // differential update. -func (bk *customTemplateAzureBlob) Update(location string, ctx context.Context) { +func (bk *customTemplateAzureBlob) Update(ctx context.Context) { // Treat the update as a download of all templates from the container - bk.Download(location, ctx) + bk.Download(ctx) } // downloadTemplate downloads a template from the Azure Blob Storage container to the local filesystem with the provided diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index 03168e5837..e0cd75d9ce 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -10,11 +10,15 @@ import ( "github.com/google/go-github/github" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" fileutil "github.com/projectdiscovery/utils/file" "golang.org/x/oauth2" "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) +var _ Provider = &customTemplateGithubRepo{} + type customTemplateGithubRepo struct { owner string reponame string @@ -23,9 +27,8 @@ type customTemplateGithubRepo struct { } // This function download the custom github template repository -func (customTemplate *customTemplateGithubRepo) Download(location string, ctx context.Context) { - downloadPath := filepath.Join(location, CustomGithubTemplateDirectory) - clonePath := customTemplate.getLocalRepoClonePath(downloadPath) +func (customTemplate *customTemplateGithubRepo) Download(ctx context.Context) { + clonePath := customTemplate.getLocalRepoClonePath(config.DefaultConfig.CustomGithubTemplatesDirectory) if !fileutil.FolderExists(clonePath) { err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken) @@ -38,13 +41,13 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co } } -func (customTemplate *customTemplateGithubRepo) Update(location string, ctx context.Context) { - downloadPath := filepath.Join(location, CustomGithubTemplateDirectory) +func (customTemplate *customTemplateGithubRepo) Update(ctx context.Context) { + downloadPath := config.DefaultConfig.CustomGithubTemplatesDirectory clonePath := customTemplate.getLocalRepoClonePath(downloadPath) // If folder does not exits then clone/download the repo if !fileutil.FolderExists(clonePath) { - customTemplate.Download(location, ctx) + customTemplate.Download(ctx) return } err := customTemplate.pullChanges(clonePath, customTemplate.githubToken) @@ -55,6 +58,33 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont } } +// NewGithubProviders returns new instance of github providers for downloading custom templates +func NewGithubProviders(options *types.Options) ([]*customTemplateGithubRepo, error) { + providers := []*customTemplateGithubRepo{} + gitHubClient := getGHClientIncognito() + + for _, repoName := range options.GithubTemplateRepo { + owner, repo, err := getOwnerAndRepo(repoName) + if err != nil { + gologger.Error().Msgf("%s", err) + continue + } + githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken) + if err != nil { + gologger.Error().Msgf("%s", err) + continue + } + customTemplateRepo := &customTemplateGithubRepo{ + owner: owner, + reponame: repo, + gitCloneURL: githubRepo.GetCloneURL(), + githubToken: options.GithubToken, + } + providers = append(providers, customTemplateRepo) + } + return providers, nil +} + // getOwnerAndRepo returns the owner, repo, err from the given string // eg. it takes input projectdiscovery/nuclei-templates and // returns owner=> projectdiscovery , repo => nuclei-templates diff --git a/v2/pkg/external/customtemplates/github_test.go b/v2/pkg/external/customtemplates/github_test.go index f52c2551f4..60d469565a 100644 --- a/v2/pkg/external/customtemplates/github_test.go +++ b/v2/pkg/external/customtemplates/github_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/stretchr/testify/require" ) @@ -18,14 +19,16 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(templatesDirectory) + config.DefaultConfig.SetTemplatesDir(templatesDirectory) + options := testutils.DefaultOptions options.GithubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"} options.GithubToken = os.Getenv("GITHUB_TOKEN") - customTemplates := ParseCustomTemplates(options) - for _, ct := range customTemplates { - ct.Download(templatesDirectory, context.Background()) - } + ctm, err := NewCustomTemplatesManager(options) + require.Nil(t, err, "could not create custom templates manager") + + ctm.Download(context.Background()) require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists") require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists") diff --git a/v2/pkg/external/customtemplates/gitlab.go b/v2/pkg/external/customtemplates/gitlab.go new file mode 100644 index 0000000000..ad502bcbab --- /dev/null +++ b/v2/pkg/external/customtemplates/gitlab.go @@ -0,0 +1,144 @@ +package customtemplates + +import ( + "context" + "encoding/base64" + "os" + "path/filepath" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" + "github.com/xanzy/go-gitlab" +) + +var _ Provider = &customTemplateGitLabRepo{} + +type customTemplateGitLabRepo struct { + gitLabClient *gitlab.Client + serverURL string + projectIDs []int +} + +// NewGitlabProviders returns a new list of GitLab providers for downloading custom templates +func NewGitlabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) { + providers := []*customTemplateGitLabRepo{} + if options.GitLabToken != "" { + // Establish a connection to GitLab and build a client object with which to download templates from GitLab + gitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("Error establishing GitLab client for %s %s", options.GitLabServerURL, err) + } + + // Create a new GitLab service client + gitLabContainer := &customTemplateGitLabRepo{ + gitLabClient: gitLabClient, + serverURL: options.GitLabServerURL, + projectIDs: options.GitLabTemplateRepositoryIDs, + } + + // Add the GitLab service client to the list of custom templates + providers = append(providers, gitLabContainer) + } + return providers, nil +} + +// Download downloads all .yaml files from a GitLab repository +func (bk *customTemplateGitLabRepo) Download(_ context.Context) { + + // Define the project and template count + var projectCount = 0 + var templateCount = 0 + + // Append the GitLab directory to the location + location := config.DefaultConfig.CustomGitLabTemplatesDirectory + + // Ensure the CustomGitLabTemplateDirectory directory exists or create it if it doesn't yet exist + err := os.MkdirAll(filepath.Dir(location), 0755) + if err != nil { + gologger.Error().Msgf("Error creating directory: %v", err) + return + } + + // Get the projects from the GitLab serverURL + for _, projectID := range bk.projectIDs { + + // Get the project information from the GitLab serverURL to get the default branch and the project name + project, _, err := bk.gitLabClient.Projects.GetProject(projectID, nil) + if err != nil { + gologger.Error().Msgf("error retrieving GitLab project: %s %s", project, err) + return + } + + // Add a subdirectory with the project ID as the subdirectory within the location + projectOutputPath := filepath.Join(location, project.Path) + + // Ensure the subdirectory exists or create it if it doesn't yet exist + err = os.MkdirAll(projectOutputPath, 0755) + if err != nil { + gologger.Error().Msgf("Error creating subdirectory: %v", err) + return + } + + // Get the directory listing for the files in the project + tree, _, err := bk.gitLabClient.Repositories.ListTree(projectID, &gitlab.ListTreeOptions{ + Ref: gitlab.String(project.DefaultBranch), + Recursive: gitlab.Bool(true), + }) + if err != nil { + gologger.Error().Msgf("error retrieving files from GitLab project: %s (%d) %s", project.Name, projectID, err) + } + + // Loop through the tree and download the files + for _, file := range tree { + // If the object is not a file or file extension is not .yaml, skip it + if file.Type == "blob" && filepath.Ext(file.Path) == ".yaml" { + gf := &gitlab.GetFileOptions{ + Ref: gitlab.String(project.DefaultBranch), + } + f, _, err := bk.gitLabClient.RepositoryFiles.GetFile(projectID, file.Path, gf) + if err != nil { + gologger.Error().Msgf("error retrieving GitLab project file: %d %s", projectID, err) + return + } + + // Decode the file content from base64 into bytes so that it can be written to the local filesystem + contents, err := base64.StdEncoding.DecodeString(f.Content) + if err != nil { + gologger.Error().Msgf("error decoding GitLab project (%s) file: %s %s", project.Name, f.FileName, err) + return + } + + // Write the downloaded template to the local filesystem at the location with the filename of the blob name + err = os.WriteFile(filepath.Join(projectOutputPath, f.FileName), contents, 0644) + if err != nil { + gologger.Error().Msgf("error writing GitLab project (%s) file: %s %s", project.Name, f.FileName, err) + return + } + + // Increment the number of templates downloaded + templateCount++ + } + } + + // Increment the number of projects downloaded + projectCount++ + gologger.Info().Msgf("GitLab project '%s' (%d) cloned successfully", project.Name, projectID) + } + + // Print the number of projects and templates downloaded + gologger.Info().Msgf("%d templates downloaded from %d GitLab project(s) to: %s", templateCount, projectCount, location) +} + +// Update is a wrapper around Download since it doesn't maintain a diff of the templates downloaded versus in the +// repository for simplicity. +func (bk *customTemplateGitLabRepo) Update(ctx context.Context) { + bk.Download(ctx) +} + +// getGitLabClient returns a GitLab client for the given serverURL and token +func getGitLabClient(server string, token string) (*gitlab.Client, error) { + client, err := gitlab.NewClient(token, gitlab.WithBaseURL(server)) + return client, err +} diff --git a/v2/pkg/external/customtemplates/s3.go b/v2/pkg/external/customtemplates/s3.go index 674fceb0d5..0804590d28 100644 --- a/v2/pkg/external/customtemplates/s3.go +++ b/v2/pkg/external/customtemplates/s3.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -11,9 +12,14 @@ import ( "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/projectdiscovery/gologger" + nucleiConfig "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" stringsutil "github.com/projectdiscovery/utils/strings" ) +var _ Provider = &customTemplateS3Bucket{} + type customTemplateS3Bucket struct { s3Client *s3.Client bucketName string @@ -22,8 +28,8 @@ type customTemplateS3Bucket struct { } // Download retrieves all custom templates from s3 bucket -func (bk *customTemplateS3Bucket) Download(location string, ctx context.Context) { - downloadPath := filepath.Join(location, CustomS3TemplateDirectory, bk.bucketName) +func (bk *customTemplateS3Bucket) Download(ctx context.Context) { + downloadPath := filepath.Join(nucleiConfig.DefaultConfig.CustomS3TemplatesDirectory, bk.bucketName) s3Manager := manager.NewDownloader(bk.s3Client) paginator := s3.NewListObjectsV2Paginator(bk.s3Client, &s3.ListObjectsV2Input{ @@ -47,9 +53,31 @@ func (bk *customTemplateS3Bucket) Download(location string, ctx context.Context) gologger.Info().Msgf("AWS bucket %s was cloned successfully at %s", bk.bucketName, downloadPath) } -// Update download custom templates from s3 bucket -func (bk *customTemplateS3Bucket) Update(location string, ctx context.Context) { - bk.Download(location, ctx) +// Update downloads custom templates from s3 bucket +func (bk *customTemplateS3Bucket) Update(ctx context.Context) { + bk.Download(ctx) +} + +// NewS3Providers returns a new instances of a s3 providers for downloading custom templates +func NewS3Providers(options *types.Options) ([]*customTemplateS3Bucket, error) { + providers := []*customTemplateS3Bucket{} + if options.AwsBucketName != "" { + s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("error downloading s3 bucket %s", options.AwsBucketName) + } + ctBucket := &customTemplateS3Bucket{ + bucketName: options.AwsBucketName, + s3Client: s3c, + } + if strings.Contains(options.AwsBucketName, "/") { + bPath := strings.SplitN(options.AwsBucketName, "/", 2) + ctBucket.bucketName = bPath[0] + ctBucket.prefix = bPath[1] + } + providers = append(providers, ctBucket) + } + return providers, nil } func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key string) error { diff --git a/v2/pkg/external/customtemplates/templates_provider.go b/v2/pkg/external/customtemplates/templates_provider.go index ebf790c74d..d016854ac3 100644 --- a/v2/pkg/external/customtemplates/templates_provider.go +++ b/v2/pkg/external/customtemplates/templates_provider.go @@ -2,84 +2,79 @@ package customtemplates import ( "context" - "strings" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -const ( - CustomGithubTemplateDirectory = "github" - CustomS3TemplateDirectory = "s3" - CustomAzureTemplateDirectory = "azure" + errorutil "github.com/projectdiscovery/utils/errors" ) type Provider interface { - Download(location string, ctx context.Context) - Update(location string, ctx context.Context) + Download(ctx context.Context) + Update(ctx context.Context) +} + +// CustomTemplatesManager is a manager for custom templates +type CustomTemplatesManager struct { + providers []Provider +} + +// Download downloads the custom templates +func (c *CustomTemplatesManager) Download(ctx context.Context) { + for _, provider := range c.providers { + provider.Download(ctx) + } +} + +// Update updates the custom templates +func (c *CustomTemplatesManager) Update(ctx context.Context) { + for _, provider := range c.providers { + provider.Update(ctx) + } } -// ParseCustomTemplates function reads the options.GithubTemplateRepo list, -// Checks the given repos are valid or not and stores them into runner.CustomTemplates -func ParseCustomTemplates(options *types.Options) []Provider { +// NewCustomTemplatesManager returns a new instance of a custom templates manager +func NewCustomTemplatesManager(options *types.Options) (*CustomTemplatesManager, error) { + ctm := &CustomTemplatesManager{providers: []Provider{}} + if options.Cloud { - return nil + // if cloud is enabled, custom templates are Nop + return ctm, nil } - var customTemplates []Provider - gitHubClient := getGHClientIncognito() - for _, repoName := range options.GithubTemplateRepo { - owner, repo, err := getOwnerAndRepo(repoName) - if err != nil { - gologger.Error().Msgf("%s", err) - continue - } - githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken) - if err != nil { - gologger.Error().Msgf("%s", err) - continue - } - customTemplateRepo := &customTemplateGithubRepo{ - owner: owner, - reponame: repo, - gitCloneURL: githubRepo.GetCloneURL(), - githubToken: options.GithubToken, - } - customTemplates = append(customTemplates, customTemplateRepo) + // Add github providers + githubProviders, err := NewGithubProviders(options) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create github providers for custom templates") } - if options.AwsBucketName != "" { - s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion) - if err != nil { - gologger.Error().Msgf("error downloading s3 bucket %s %s", options.AwsBucketName, err) - return customTemplates - } - ctBucket := &customTemplateS3Bucket{ - bucketName: options.AwsBucketName, - s3Client: s3c, - } - if strings.Contains(options.AwsBucketName, "/") { - bPath := strings.SplitN(options.AwsBucketName, "/", 2) - ctBucket.bucketName = bPath[0] - ctBucket.prefix = bPath[1] - } - customTemplates = append(customTemplates, ctBucket) + for _, v := range githubProviders { + ctm.providers = append(ctm.providers, v) } - if options.AzureContainerName != "" { - // Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage - azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL) - if err != nil { - gologger.Error().Msgf("Error establishing Azure Blob client for %s %s", options.AzureContainerName, err) - return customTemplates - } - // Create a new Azure Blob Storage container object - azTemplateContainer := &customTemplateAzureBlob{ - azureBlobClient: azClient, - containerName: options.AzureContainerName, - } + // Add Aws S3 providers + s3Providers, err := NewS3Providers(options) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create s3 providers for custom templates") + } + for _, v := range s3Providers { + ctm.providers = append(ctm.providers, v) + } - // Add the Azure Blob Storage container object to the list of custom templates - customTemplates = append(customTemplates, azTemplateContainer) + // Add Azure providers + azureProviders, err := NewAzureProviders(options) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create azure providers for custom templates") + } + for _, v := range azureProviders { + ctm.providers = append(ctm.providers, v) } - return customTemplates + + // Add GitLab providers + gitlabProviders, err := NewGitlabProviders(options) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create gitlab providers for custom templates") + } + for _, v := range gitlabProviders { + ctm.providers = append(ctm.providers, v) + } + + return ctm, nil } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 0cae038945..3a5b7fff4d 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -326,17 +326,23 @@ type Options struct { ScanAllIPs bool // IPVersion to scan (4,6) IPVersion goflags.StringSlice - // Github token used to clone/pull from private repos for custom templates + // GitHub token used to clone/pull from private repos for custom templates GithubToken string - // GithubTemplateRepo is the list of custom public/private templates github repos + // GithubTemplateRepo is the list of custom public/private templates GitHub repos GithubTemplateRepo []string - // AWS access key for downloading templates from s3 bucket + // GitLabServerURL is the gitlab server to use for custom templates + GitLabServerURL string + // GitLabToken used to clone/pull from private repos for custom templates + GitLabToken string + // GitLabTemplateRepositoryIDs is the comma-separated list of custom gitlab repositories IDs + GitLabTemplateRepositoryIDs []int + // AWS access key for downloading templates from S3 bucket AwsAccessKey string - // AWS secret key for downloading templates from s3 bucket + // AWS secret key for downloading templates from S3 bucket AwsSecretKey string - // AWS bucket name for downloading templates from s3 bucket + // AWS bucket name for downloading templates from S3 bucket AwsBucketName string - // AWS Region name where aws s3 bucket is located + // AWS Region name where AWS S3 bucket is located AwsRegion string // AzureContainerName for downloading templates from Azure Blob Storage. Example: templates AzureContainerName string From 871e7016fcb348dd36c096ddb1898f28975ef517 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 20 Apr 2023 03:39:33 +0530 Subject: [PATCH 34/34] version update --- README.md | 2 +- v2/cmd/nuclei/main.go | 2 +- v2/pkg/catalog/config/constants.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5e3571fad6..3adb40e769 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,7 @@ DEBUG: -hc, -health-check run diagnostic check up UPDATE: - -un, -update update nuclei engine to the latest released version + -up, -update update nuclei engine to the latest released version -ut, -update-templates update nuclei-templates to latest released version -ud, -update-template-dir string custom directory to install / update nuclei-templates -duc, -disable-update-check disable automatic nuclei/templates update check diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 20b86dbe68..f39cd74562 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -292,7 +292,7 @@ on extensive configurability, massive extensibility and ease of use.`) ) flagSet.CreateGroup("update", "Update", - flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "un", "update nuclei engine to the latest released version"), + flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "up", "update nuclei engine to the latest released version"), flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"), flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"), flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"), diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 9e4a06e29d..493671f62f 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -16,7 +16,7 @@ const ( CLIConifgFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v2.9.2-dev` + Version = `v2.9.2` // Directory Names of custom templates CustomS3TemplatesDirName = "s3"