From 1353caab0fcce42813250590a0b0e038f459c158 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 16:04:02 +0000 Subject: [PATCH 01/58] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.7 to 1.0.8. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.7...v1.0.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index f0bd3828..f18713b5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/gologger v1.1.5 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.4 - github.com/projectdiscovery/retryablehttp-go v1.0.7 + github.com/projectdiscovery/retryablehttp-go v1.0.8 github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 ) @@ -41,9 +41,9 @@ require ( github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/net v0.4.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/net v0.5.0 // indirect + golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8633ee40..3f4783ab 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.4 h1:2TOAhuOUMlKrzwissru2mFnSd8eg2WddIQKcAyYEkGs= github.com/projectdiscovery/ratelimit v0.0.4/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= -github.com/projectdiscovery/retryablehttp-go v1.0.7 h1:4Nd87QN+3yUMxYwQQgG/v+BLQt1fYqXu9PjEoar8LGs= -github.com/projectdiscovery/retryablehttp-go v1.0.7/go.mod h1:ODXs70i/PgqfqFvycLfQG6QCqWqZMLoX8X68wu6Bg8M= +github.com/projectdiscovery/retryablehttp-go v1.0.8 h1:F4UZ+3NkIPDj+tobHZY+sgvDVmiVCX/CUUh/jrKq7Oc= +github.com/projectdiscovery/retryablehttp-go v1.0.8/go.mod h1:tD5TNOXKNuvqrUHHGoP6xJmVZTjSkLGsqcyD7gCsc3I= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= @@ -114,8 +114,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -128,16 +128,16 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= From fa9df79b94f8b59c6e465793ca9fee80ae4f4e03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 16:04:08 +0000 Subject: [PATCH 02/58] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.5 to 1.1.7. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.5...v1.1.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f0bd3828..fc7dae18 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goflags v0.1.6 - github.com/projectdiscovery/gologger v1.1.5 + github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.7 diff --git a/go.sum b/go.sum index 8633ee40..92f3d9ee 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4= github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= -github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= -github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= +github.com/projectdiscovery/gologger v1.1.7 h1:QeByO4NR8vlZ5ZM2XIOJARRj5WOOU5Ix35FBQ1iH5Rk= +github.com/projectdiscovery/gologger v1.1.7/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.4 h1:2TOAhuOUMlKrzwissru2mFnSd8eg2WddIQKcAyYEkGs= From 9df4265006eaafde751af0d71d20227ab4719547 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 16:16:25 +0000 Subject: [PATCH 03/58] chore(deps): bump alpine from 3.17.0 to 3.17.1 Bumps alpine from 3.17.0 to 3.17.1. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0372cc91..a155994c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.18.2-alpine3.14 AS build-env RUN go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest -FROM alpine:3.17.0 +FROM alpine:3.17.1 RUN apk add --no-cache bind-tools ca-certificates COPY --from=build-env /go/bin/uncover /usr/local/bin/uncover ENTRYPOINT ["uncover"] From 0c91f696d1ebc533bc934a3c65ed69f902b5503d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 16:04:41 +0000 Subject: [PATCH 04/58] chore(deps): bump golangci/golangci-lint-action from 3.3.1 to 3.4.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.3.1 to 3.4.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index a1a68c8a..39490315 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.3.1 + uses: golangci/golangci-lint-action@v3.4.0 with: version: latest args: --timeout 5m From aa800f34307b9207572a9a39604390b56b5f1c53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 16:04:58 +0000 Subject: [PATCH 05/58] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.4 to 0.0.5. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.4...v0.0.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9cc07b88..e0d13c48 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 - github.com/projectdiscovery/ratelimit v0.0.4 + github.com/projectdiscovery/ratelimit v0.0.5 github.com/projectdiscovery/retryablehttp-go v1.0.8 github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 diff --git a/go.sum b/go.sum index 0ae7fdfc..5861074b 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/projectdiscovery/gologger v1.1.7 h1:QeByO4NR8vlZ5ZM2XIOJARRj5WOOU5Ix3 github.com/projectdiscovery/gologger v1.1.7/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= -github.com/projectdiscovery/ratelimit v0.0.4 h1:2TOAhuOUMlKrzwissru2mFnSd8eg2WddIQKcAyYEkGs= -github.com/projectdiscovery/ratelimit v0.0.4/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= +github.com/projectdiscovery/ratelimit v0.0.5 h1:QAWtECH5MJtqen448fHr6ikV8NTBdebmZBLrX9oqaic= +github.com/projectdiscovery/ratelimit v0.0.5/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= github.com/projectdiscovery/retryablehttp-go v1.0.8 h1:F4UZ+3NkIPDj+tobHZY+sgvDVmiVCX/CUUh/jrKq7Oc= github.com/projectdiscovery/retryablehttp-go v1.0.8/go.mod h1:tD5TNOXKNuvqrUHHGoP6xJmVZTjSkLGsqcyD7gCsc3I= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= From 04d1a95d70ec1fa355fed43cb276cc9567658aff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 08:44:37 +0000 Subject: [PATCH 06/58] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.8 to 1.0.9. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.8...v1.0.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e0d13c48..f45d83dd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.5 - github.com/projectdiscovery/retryablehttp-go v1.0.8 + github.com/projectdiscovery/retryablehttp-go v1.0.9 github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 ) @@ -37,7 +37,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect - github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 + github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index 5861074b..9ef4deab 100644 --- a/go.sum +++ b/go.sum @@ -70,13 +70,13 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.5 h1:QAWtECH5MJtqen448fHr6ikV8NTBdebmZBLrX9oqaic= github.com/projectdiscovery/ratelimit v0.0.5/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= -github.com/projectdiscovery/retryablehttp-go v1.0.8 h1:F4UZ+3NkIPDj+tobHZY+sgvDVmiVCX/CUUh/jrKq7Oc= -github.com/projectdiscovery/retryablehttp-go v1.0.8/go.mod h1:tD5TNOXKNuvqrUHHGoP6xJmVZTjSkLGsqcyD7gCsc3I= +github.com/projectdiscovery/retryablehttp-go v1.0.9 h1:pzSmLQR++5Y4Hprxko4ZBPuXgw6yy3aUn+fe6+mgOjA= +github.com/projectdiscovery/retryablehttp-go v1.0.9/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 h1:/fZvw6gT1fzdmMLMBBw75OrJ0Z6g7dulQrxM9FRp1qU= -github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d h1:iB/n2/NL4oh1IaEcqX6pBxj0WHfYN7finzNOKVNVISM= +github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= From 019c7288d28edcd6b718eee5869bc79d2be91233 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:06:03 +0000 Subject: [PATCH 07/58] chore(deps): bump docker/build-push-action from 3 to 4 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 3 to 4. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index b689a22e..72f852d6 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -32,7 +32,7 @@ jobs: password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: context: . platforms: linux/amd64,linux/arm64,linux/arm From 9ab400daf3c4af463ec949cb205433a0064d9b5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:06:17 +0000 Subject: [PATCH 08/58] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.5 to 0.0.6. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.5...v0.0.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f45d83dd..b187a58f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 - github.com/projectdiscovery/ratelimit v0.0.5 + github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/retryablehttp-go v1.0.9 github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 @@ -37,7 +37,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect - github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d + github.com/projectdiscovery/utils v0.0.6 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index 9ef4deab..70441bd8 100644 --- a/go.sum +++ b/go.sum @@ -68,15 +68,15 @@ github.com/projectdiscovery/gologger v1.1.7 h1:QeByO4NR8vlZ5ZM2XIOJARRj5WOOU5Ix3 github.com/projectdiscovery/gologger v1.1.7/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= -github.com/projectdiscovery/ratelimit v0.0.5 h1:QAWtECH5MJtqen448fHr6ikV8NTBdebmZBLrX9oqaic= -github.com/projectdiscovery/ratelimit v0.0.5/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= +github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= +github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.9 h1:pzSmLQR++5Y4Hprxko4ZBPuXgw6yy3aUn+fe6+mgOjA= github.com/projectdiscovery/retryablehttp-go v1.0.9/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d h1:iB/n2/NL4oh1IaEcqX6pBxj0WHfYN7finzNOKVNVISM= -github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.6 h1:6SDn/5E5NxrAfcYrZ7omXPmiU9n8p0rKXZ4BAOQyzbw= +github.com/projectdiscovery/utils v0.0.6/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= From 53c76ef258ba908f06aa41e665ff2a3ee7757ae5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Feb 2023 10:29:07 +0000 Subject: [PATCH 09/58] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.9 to 1.0.10. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.9...v1.0.10) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b187a58f..46e5fcc4 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.9 + github.com/projectdiscovery/retryablehttp-go v1.0.10 github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 ) diff --git a/go.sum b/go.sum index 70441bd8..aa5d420d 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/retryablehttp-go v1.0.9 h1:pzSmLQR++5Y4Hprxko4ZBPuXgw6yy3aUn+fe6+mgOjA= -github.com/projectdiscovery/retryablehttp-go v1.0.9/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= +github.com/projectdiscovery/retryablehttp-go v1.0.10 h1:A8tsJZhzpijgnvHT41JNrhiXtn8F8s7TWMC5ZxLV/Cg= +github.com/projectdiscovery/retryablehttp-go v1.0.10/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= From 362febc933a640becddbb15d4a1cf279a47f718c Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 15 Feb 2023 18:27:40 +0100 Subject: [PATCH 10/58] started job --- uncover/agent/publicwww/publicwww.go | 38 ++++++++++++++++++++++++++++ uncover/keys.go | 19 +++++++------- 2 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 uncover/agent/publicwww/publicwww.go diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go new file mode 100644 index 00000000..2bd95610 --- /dev/null +++ b/uncover/agent/publicwww/publicwww.go @@ -0,0 +1,38 @@ +package publicwww + +import ( + "errors" + + "github.com/projectdiscovery/uncover/uncover" +) + +const ( + baseURL = "https://publicwww.com/" + baseEndpoint = "websites/" +) + +type Agent struct { + options *uncover.AgentOptions +} + +func New() (uncover.Agent, error) { + return &Agent{}, nil +} + +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + +func (agent *Agent) Name() string { + return "publicwww" +} + +func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan uncover.Result, error) { + if session.Keys.NetlasToken == "" { + return nil, errors.New("empty netlas keys") + } + + results := make(chan uncover.Result) + + return results, nil +} diff --git a/uncover/keys.go b/uncover/keys.go index 60fa2226..b2fae2e9 100644 --- a/uncover/keys.go +++ b/uncover/keys.go @@ -1,16 +1,17 @@ package uncover type Keys struct { - CensysToken string - CensysSecret string - Shodan string - FofaEmail string - FofaKey string - QuakeToken string - HunterToken string - ZoomEyeToken string - NetlasToken string + CensysToken string + CensysSecret string + Shodan string + FofaEmail string + FofaKey string + QuakeToken string + HunterToken string + ZoomEyeToken string + NetlasToken string CriminalIPToken string + PublicwwwToken string } func (keys Keys) Empty() bool { From bb1cf52a51545d5968869cb7687098f41f2a5deb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 23:00:31 +0530 Subject: [PATCH 11/58] chore(deps): bump github.com/projectdiscovery/retryablehttp-go (#156) Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.10 to 1.0.11. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.10...v1.0.11) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 46e5fcc4..b7609907 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.10 + github.com/projectdiscovery/retryablehttp-go v1.0.11 github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 ) @@ -37,7 +37,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect - github.com/projectdiscovery/utils v0.0.6 + github.com/projectdiscovery/utils v0.0.7 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index aa5d420d..09a12ffd 100644 --- a/go.sum +++ b/go.sum @@ -70,13 +70,13 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/retryablehttp-go v1.0.10 h1:A8tsJZhzpijgnvHT41JNrhiXtn8F8s7TWMC5ZxLV/Cg= -github.com/projectdiscovery/retryablehttp-go v1.0.10/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= +github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo= +github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.6 h1:6SDn/5E5NxrAfcYrZ7omXPmiU9n8p0rKXZ4BAOQyzbw= -github.com/projectdiscovery/utils v0.0.6/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDkst9DA2M= +github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= From fe6daff4c337262a9f7a89d978049d0ebe70bfa8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 23:00:50 +0530 Subject: [PATCH 12/58] chore(deps): bump github.com/projectdiscovery/gologger (#153) Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.7 to 1.1.8. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.7...v1.1.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b7609907..5a0f6d2e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goflags v0.1.6 - github.com/projectdiscovery/gologger v1.1.7 + github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/retryablehttp-go v1.0.11 diff --git a/go.sum b/go.sum index 09a12ffd..d68fcaae 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4= github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= -github.com/projectdiscovery/gologger v1.1.7 h1:QeByO4NR8vlZ5ZM2XIOJARRj5WOOU5Ix35FBQ1iH5Rk= -github.com/projectdiscovery/gologger v1.1.7/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= +github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= +github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= From 4fa664af7a5fe470c6c3ef5552d1475e9904efa4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 23:03:24 +0530 Subject: [PATCH 13/58] chore(deps): bump github.com/projectdiscovery/mapcidr (#154) Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 1.0.3 to 1.1.0. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/master/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v1.0.3...v1.1.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 9 ++++----- go.sum | 18 ++++++++---------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 5a0f6d2e..a09e70c4 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.8 - github.com/projectdiscovery/mapcidr v1.0.3 + github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/retryablehttp-go v1.0.11 github.com/projectdiscovery/stringsutil v0.0.2 @@ -19,8 +19,7 @@ require ( require ( github.com/miekg/dns v1.1.50 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 // indirect + golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 // indirect golang.org/x/mod v0.6.0 // indirect golang.org/x/tools v0.2.0 // indirect ) @@ -36,7 +35,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect + github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect github.com/projectdiscovery/utils v0.0.7 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect @@ -50,7 +49,7 @@ require ( require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/dsnet/compress v0.0.1 // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect diff --git a/go.sum b/go.sum index d68fcaae..eba539a6 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -58,27 +58,25 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= -github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= +github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ/zkotVodSJLOch3pV/UYt1vQcwyIs5HX0sm1ljE= +github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4= github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= -github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= +github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oTFX/xzGhVU= +github.com/projectdiscovery/mapcidr v1.1.0/go.mod h1:hck0bWXka5ZkUaBG+TWt99bzLy+4hAg9oANhEmm3GNs= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo= github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= -github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDkst9DA2M= github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -104,8 +102,8 @@ go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws= +golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= From f7e05ac351cbf1f470261047257c6adfe86884a7 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 15 Feb 2023 23:08:43 +0100 Subject: [PATCH 14/58] wip: publicwww integration --- runner/options.go | 6 ++- runner/provider.go | 25 ++++++---- runner/runner.go | 21 +++++--- uncover/agent/publicwww/publicwww.go | 74 +++++++++++++++++++++++++++- uncover/agent/publicwww/request.go | 10 ++++ uncover/keys.go | 3 +- 6 files changed, 118 insertions(+), 21 deletions(-) create mode 100644 uncover/agent/publicwww/request.go diff --git a/runner/options.go b/runner/options.go index 5e2047f3..1cb95f87 100644 --- a/runner/options.go +++ b/runner/options.go @@ -50,6 +50,7 @@ type Options struct { Hunter goflags.StringSlice ZoomEye goflags.StringSlice CriminalIP goflags.StringSlice + Publicwww goflags.StringSlice } // ParseOptions parses the command line flags provided by a user @@ -74,6 +75,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.ZoomEye, "zoomeye", "ze", nil, "search query for zoomeye (example: -zoomeye 'query.txt')", goflags.FileStringSliceOptions), flagSet.StringSliceVarP(&options.Netlas, "netlas", "ne", nil, "search query for netlas (example: -netlas 'query.txt')", goflags.FileStringSliceOptions), flagSet.StringSliceVarP(&options.CriminalIP, "criminalip", "cl", nil, "search query for criminalip (example: -criminalip 'query.txt')", goflags.FileStringSliceOptions), + flagSet.StringSliceVarP(&options.Publicwww, "publicwww", "pw", nil, "search query for publicwww (example: -publicwww 'query.txt')", goflags.FileStringSliceOptions), ) flagSet.CreateGroup("config", "Config", @@ -217,8 +219,8 @@ func (options *Options) loadProvidersFromEnv() error { options.Provider.Netlas = append(options.Provider.Netlas, key) } if key, exists := os.LookupEnv("CRIMINALIP_API_KEY"); exists { - options.Provider.CriminalIP = append(options.Provider.CriminalIP, key) - } + options.Provider.CriminalIP = append(options.Provider.CriminalIP, key) + } return nil } diff --git a/runner/provider.go b/runner/provider.go index 072f09ac..341df3fa 100644 --- a/runner/provider.go +++ b/runner/provider.go @@ -8,14 +8,15 @@ import ( ) type Provider struct { - Shodan []string `yaml:"shodan"` - Censys []string `yaml:"censys"` - Fofa []string `yaml:"fofa"` - Quake []string `yaml:"quake"` - Hunter []string `yaml:"hunter"` - ZoomEye []string `yaml:"zoomeye"` - Netlas []string `yaml:"netlas"` + Shodan []string `yaml:"shodan"` + Censys []string `yaml:"censys"` + Fofa []string `yaml:"fofa"` + Quake []string `yaml:"quake"` + Hunter []string `yaml:"hunter"` + ZoomEye []string `yaml:"zoomeye"` + Netlas []string `yaml:"netlas"` CriminalIP []string `yaml:"criminalip"` + Publicwww []string `yaml:"publicwww"` } func (provider *Provider) GetKeys() uncover.Keys { @@ -59,9 +60,13 @@ func (provider *Provider) GetKeys() uncover.Keys { keys.NetlasToken = provider.Netlas[rand.Intn(len(provider.Netlas))] } - if len(provider.CriminalIP) > 0 { - keys.CriminalIPToken = provider.CriminalIP[rand.Intn(len(provider.CriminalIP))] - } + if len(provider.CriminalIP) > 0 { + keys.CriminalIPToken = provider.CriminalIP[rand.Intn(len(provider.CriminalIP))] + } + + if len(provider.Publicwww) > 0 { + keys.PublicwwwToken = provider.Publicwww[rand.Intn(len(provider.Publicwww))] + } return keys } diff --git a/runner/runner.go b/runner/runner.go index 43afc69b..d84b0879 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -15,14 +15,15 @@ import ( "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/uncover/uncover" "github.com/projectdiscovery/uncover/uncover/agent/censys" + "github.com/projectdiscovery/uncover/uncover/agent/criminalip" "github.com/projectdiscovery/uncover/uncover/agent/fofa" "github.com/projectdiscovery/uncover/uncover/agent/hunter" "github.com/projectdiscovery/uncover/uncover/agent/netlas" + "github.com/projectdiscovery/uncover/uncover/agent/publicwww" "github.com/projectdiscovery/uncover/uncover/agent/quake" "github.com/projectdiscovery/uncover/uncover/agent/shodan" "github.com/projectdiscovery/uncover/uncover/agent/shodanidb" "github.com/projectdiscovery/uncover/uncover/agent/zoomeye" - "github.com/projectdiscovery/uncover/uncover/agent/criminalip" ) func init() { @@ -49,7 +50,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return errors.New("no keys provided") } - var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter *ratelimit.Limiter + var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter, publicwwwRatelimiter *ratelimit.Limiter if r.options.Delay > 0 { censysRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) fofaRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) @@ -60,6 +61,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { zoomeyeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) netlasRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) criminalipRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + publicwwwRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) } else { censysRateLimiter = ratelimit.NewUnlimited(context.Background()) fofaRateLimiter = ratelimit.NewUnlimited(context.Background()) @@ -70,6 +72,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { zoomeyeRatelimiter = ratelimit.NewUnlimited(context.Background()) netlasRatelimiter = ratelimit.NewUnlimited(context.Background()) criminalipRatelimiter = ratelimit.NewUnlimited(context.Background()) + publicwwwRatelimiter = ratelimit.NewUnlimited(context.Background()) } var agents []uncover.Agent @@ -106,9 +109,13 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { query = append(query, r.options.Netlas...) } if len(r.options.CriminalIP) > 0 { - r.options.Engine = append(r.options.Engine, "criminalip") - query = append(query, r.options.CriminalIP...) - } + r.options.Engine = append(r.options.Engine, "criminalip") + query = append(query, r.options.CriminalIP...) + } + if len(r.options.Publicwww) > 0 { + r.options.Engine = append(r.options.Engine, "publicwww") + query = append(query, r.options.Publicwww...) + } // declare clients for _, engine := range r.options.Engine { @@ -134,7 +141,9 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { case "netlas": agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: netlasRatelimiter}) case "criminalip": - agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: criminalipRatelimiter}) + agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: criminalipRatelimiter}) + case "publicwww": + agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{RateLimiter: publicwwwRatelimiter}) default: err = errors.New("unknown agent type") } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 2bd95610..e2b25754 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -2,6 +2,9 @@ package publicwww import ( "errors" + "io/ioutil" + "net/http" + "strings" "github.com/projectdiscovery/uncover/uncover" ) @@ -28,11 +31,78 @@ func (agent *Agent) Name() string { } func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan uncover.Result, error) { - if session.Keys.NetlasToken == "" { - return nil, errors.New("empty netlas keys") + + if session.Keys.PublicwwwToken == "" { + return nil, errors.New("empty publicwww keys") } results := make(chan uncover.Result) + go func() { + defer close(results) + + numberOfResults := 0 + + for { + publicwwwRequest := &Request{ + Query: query.Query, + Start: numberOfResults, + } + + publicwwwResponse := agent.query(publicwwwRequest.buildURL(), session, results) + if publicwwwResponse == nil { + break + } + + if numberOfResults > query.Limit || len(publicwwwResponse) == 0 { + break + } + + numberOfResults += len(publicwwwResponse) + } + }() + return results, nil } + +func (agent *Agent) query(URL string, session *uncover.Session, results chan uncover.Result) []string { + resp, err := agent.queryURL(session, URL) + if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} + return nil + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} + return nil + } + content := string(body) + + var lines []string + for _, line := range strings.Split(content, "\n") { + result := uncover.Result{Source: agent.Name()} + result.Host = line + results <- result + trimmedLine := strings.TrimRight(line, " \r\n\t") + if trimmedLine != "" { + lines = append(lines, trimmedLine) + } + } + + return lines +} + +func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { + request, err := uncover.NewHTTPRequest( + http.MethodGet, + URL, + nil, + ) + if err != nil { + return nil, err + } + + agent.options.RateLimiter.Take() + return session.Do(request) +} diff --git a/uncover/agent/publicwww/request.go b/uncover/agent/publicwww/request.go new file mode 100644 index 00000000..47f57c7a --- /dev/null +++ b/uncover/agent/publicwww/request.go @@ -0,0 +1,10 @@ +package publicwww + +type Request struct { + Query string `json:"query"` + Start int `json:"start"` +} + +func (r *Request) buildURL() string { + return "" +} diff --git a/uncover/keys.go b/uncover/keys.go index b2fae2e9..9a76a6bb 100644 --- a/uncover/keys.go +++ b/uncover/keys.go @@ -24,5 +24,6 @@ func (keys Keys) Empty() bool { keys.HunterToken == "" && keys.ZoomEyeToken == "" && keys.NetlasToken == "" && - keys.CriminalIPToken == "" + keys.CriminalIPToken == "" && + keys.PublicwwwToken == "" } From ec4c725820e2367a2c76783ab43a315e7e5dc828 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Thu, 16 Feb 2023 15:27:35 +0100 Subject: [PATCH 15/58] added build url logic --- uncover/agent/publicwww/publicwww.go | 3 +-- uncover/agent/publicwww/request.go | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index e2b25754..2d15fc5d 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -49,7 +49,7 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan Start: numberOfResults, } - publicwwwResponse := agent.query(publicwwwRequest.buildURL(), session, results) + publicwwwResponse := agent.query(publicwwwRequest.buildURL(session.Keys.PublicwwwToken), session, results) if publicwwwResponse == nil { break } @@ -78,7 +78,6 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc return nil } content := string(body) - var lines []string for _, line := range strings.Split(content, "\n") { result := uncover.Result{Source: agent.Name()} diff --git a/uncover/agent/publicwww/request.go b/uncover/agent/publicwww/request.go index 47f57c7a..d2db63e7 100644 --- a/uncover/agent/publicwww/request.go +++ b/uncover/agent/publicwww/request.go @@ -1,10 +1,15 @@ package publicwww +import "net/url" + type Request struct { Query string `json:"query"` Start int `json:"start"` } -func (r *Request) buildURL() string { - return "" +func (r *Request) buildURL(key string) string { + return baseURL + + baseEndpoint + + url.QueryEscape(`"`+r.Query+`"`) + + `/?export=urls&key=` + key } From 04b48390ca7c5533d6a1194971a517215deea67c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Feb 2023 00:18:54 +0530 Subject: [PATCH 16/58] chore(deps): bump alpine from 3.17.1 to 3.17.2 (#152) Bumps alpine from 3.17.1 to 3.17.2. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a155994c..e89a6e5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.18.2-alpine3.14 AS build-env RUN go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest -FROM alpine:3.17.1 +FROM alpine:3.17.2 RUN apk add --no-cache bind-tools ca-certificates COPY --from=build-env /go/bin/uncover /usr/local/bin/uncover ENTRYPOINT ["uncover"] From 9317c1539d598f2c75d9fc2edb81f4abdc5a2cfd Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Fri, 17 Feb 2023 12:41:58 +0100 Subject: [PATCH 17/58] results management --- uncover/agent/publicwww/publicwww.go | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 2d15fc5d..5aa0b2e0 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -46,7 +46,6 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan for { publicwwwRequest := &Request{ Query: query.Query, - Start: numberOfResults, } publicwwwResponse := agent.query(publicwwwRequest.buildURL(session.Keys.PublicwwwToken), session, results) @@ -66,11 +65,11 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan } func (agent *Agent) query(URL string, session *uncover.Session, results chan uncover.Result) []string { - resp, err := agent.queryURL(session, URL) + resp, err := http.Get(URL) if err != nil { - results <- uncover.Result{Source: agent.Name(), Error: err} return nil } + defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -81,27 +80,13 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc var lines []string for _, line := range strings.Split(content, "\n") { result := uncover.Result{Source: agent.Name()} - result.Host = line - results <- result trimmedLine := strings.TrimRight(line, " \r\n\t") if trimmedLine != "" { + result.Host = line + results <- result lines = append(lines, trimmedLine) } } return lines } - -func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { - request, err := uncover.NewHTTPRequest( - http.MethodGet, - URL, - nil, - ) - if err != nil { - return nil, err - } - - agent.options.RateLimiter.Take() - return session.Do(request) -} From f7e62a977172a0a8cdde8bf29edb217eefbfadfa Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 18 Feb 2023 14:57:26 +0100 Subject: [PATCH 18/58] small changes --- uncover/agent/publicwww/publicwww.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 5aa0b2e0..60ccc7c3 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -2,7 +2,7 @@ package publicwww import ( "errors" - "io/ioutil" + "io" "net/http" "strings" @@ -31,7 +31,6 @@ func (agent *Agent) Name() string { } func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan uncover.Result, error) { - if session.Keys.PublicwwwToken == "" { return nil, errors.New("empty publicwww keys") } @@ -71,7 +70,7 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { results <- uncover.Result{Source: agent.Name(), Error: err} return nil From 6f3e011c3a36bd5f47989a16a8210969d19cdaff Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Sat, 18 Feb 2023 22:19:38 +0100 Subject: [PATCH 19/58] wip: added url flag, used retryablehttp for request + return raw results todo, parse and use csv instead of export=urls --- runner/runner.go | 17 +++++++++++++++- uncover/agent/publicwww/publicwww.go | 29 +++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index d84b0879..bccb9423 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "net/url" "os" "strings" "sync" @@ -211,10 +212,16 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { outputWriter.WriteString(result.RawData()) default: port := fmt.Sprint(result.Port) + hostname, err := getHostname(result.Host) + if err != nil { + gologger.Warning().Msgf("%s\n", err) + return + } replacer := strings.NewReplacer( "ip", result.IP, - "host", result.Host, + "host", hostname, "port", port, + "url", result.Host, ) outData := replacer.Replace(r.options.OutputFields) searchFor := []string{result.IP, port} @@ -236,3 +243,11 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { wg.Wait() return nil } + +func getHostname(u string) (string, error) { + parsedURL, err := url.Parse(u) + if err != nil { + return "", err + } + return parsedURL.Hostname(), nil +} diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 60ccc7c3..cb4983d0 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -1,6 +1,7 @@ package publicwww import ( + "encoding/json" "errors" "io" "net/http" @@ -47,12 +48,16 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan Query: query.Query, } + if numberOfResults > query.Limit { + break + } + publicwwwResponse := agent.query(publicwwwRequest.buildURL(session.Keys.PublicwwwToken), session, results) if publicwwwResponse == nil { break } - if numberOfResults > query.Limit || len(publicwwwResponse) == 0 { + if len(publicwwwResponse) == 0 { break } @@ -64,11 +69,11 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan } func (agent *Agent) query(URL string, session *uncover.Session, results chan uncover.Result) []string { - resp, err := http.Get(URL) + resp, err := agent.queryURL(session, URL) if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} return nil } - defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { @@ -82,6 +87,8 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc trimmedLine := strings.TrimRight(line, " \r\n\t") if trimmedLine != "" { result.Host = line + raw, _ := json.Marshal(result) + result.Raw = raw results <- result lines = append(lines, trimmedLine) } @@ -89,3 +96,19 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc return lines } + +func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { + request, err := uncover.NewHTTPRequest( + http.MethodGet, + URL, + nil, + ) + if err != nil { + return nil, err + } + + request.Header.Set("Content-Type", "application/vnd.ms-excel") + + agent.options.RateLimiter.Take() + return session.Do(request) +} From 02390b04b81f5ec5a503af0299937cd80af4daf8 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 20 Feb 2023 09:11:03 +0100 Subject: [PATCH 20/58] dedicated search query and env support --- runner/options.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runner/options.go b/runner/options.go index 1cb95f87..fe8c0ee6 100644 --- a/runner/options.go +++ b/runner/options.go @@ -141,7 +141,8 @@ func ParseOptions() *Options { len(options.Hunter) == 0 && len(options.ZoomEye) == 0 && len(options.Netlas) == 0 && - len(options.CriminalIP) == 0 { + len(options.CriminalIP) == 0 && + len(options.Publicwww) == 0 { options.Engine = append(options.Engine, "shodan") } @@ -221,6 +222,9 @@ func (options *Options) loadProvidersFromEnv() error { if key, exists := os.LookupEnv("CRIMINALIP_API_KEY"); exists { options.Provider.CriminalIP = append(options.Provider.CriminalIP, key) } + if key, exists := os.LookupEnv("PUBLICWWW_API_KEY"); exists { + options.Provider.Publicwww = append(options.Provider.Publicwww, key) + } return nil } @@ -237,7 +241,8 @@ func (options *Options) validateOptions() error { len(options.Hunter) == 0 && len(options.ZoomEye) == 0 && len(options.Netlas) == 0 && - len(options.CriminalIP) == 0 { + len(options.CriminalIP) == 0 && + len(options.Publicwww) == 0 { return errors.New("no query provided") } @@ -256,7 +261,8 @@ func (options *Options) validateOptions() error { len(options.Hunter) == 0 && len(options.ZoomEye) == 0 && len(options.Netlas) == 0 && - len(options.CriminalIP) == 0 { + len(options.CriminalIP) == 0 && + len(options.Publicwww) == 0 { return errors.New("no engine specified") } From 49b1e16443d2e48eadda5dc6a817b69a1d6e8041 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Feb 2023 00:53:59 +0530 Subject: [PATCH 21/58] chore(deps): bump github.com/projectdiscovery/goflags (#158) Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.6 to 0.1.7. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.6...v0.1.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a09e70c4..1e3cff7a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/goflags v0.1.6 + github.com/projectdiscovery/goflags v0.1.7 github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 diff --git a/go.sum b/go.sum index eba539a6..b3db6344 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4= -github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= +github.com/projectdiscovery/goflags v0.1.7 h1:KvMs1KnXMWRhJiJj6K+cWurTWpbDYYlo+1cyGvf1YCc= +github.com/projectdiscovery/goflags v0.1.7/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oTFX/xzGhVU= From dad8e072caa9840cfee8c8fb10307597854cd90c Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 20 Feb 2023 21:44:09 +0100 Subject: [PATCH 22/58] fixes and csv export --- runner/runner.go | 18 ++---------------- uncover/agent/publicwww/publicwww.go | 23 ++++++++++++++++++++--- uncover/agent/publicwww/request.go | 6 ++---- uncover/result.go | 1 + uncover/util.go | 10 ++++++++++ 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index bccb9423..85a7c4e7 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/rand" - "net/url" "os" "strings" "sync" @@ -212,16 +211,11 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { outputWriter.WriteString(result.RawData()) default: port := fmt.Sprint(result.Port) - hostname, err := getHostname(result.Host) - if err != nil { - gologger.Warning().Msgf("%s\n", err) - return - } replacer := strings.NewReplacer( "ip", result.IP, - "host", hostname, + "host", result.Host, "port", port, - "url", result.Host, + "url", result.Url, ) outData := replacer.Replace(r.options.OutputFields) searchFor := []string{result.IP, port} @@ -243,11 +237,3 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { wg.Wait() return nil } - -func getHostname(u string) (string, error) { - parsedURL, err := url.Parse(u) - if err != nil { - return "", err - } - return parsedURL.Hostname(), nil -} diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index cb4983d0..2b54696c 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -1,6 +1,7 @@ package publicwww import ( + "encoding/csv" "encoding/json" "errors" "io" @@ -81,12 +82,28 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc return nil } content := string(body) + reader := csv.NewReader(strings.NewReader(content)) + reader.Comma = ';' + var lines []string - for _, line := range strings.Split(content, "\n") { + for { + record, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + results <- uncover.Result{Source: agent.Name(), Error: err} + } + result := uncover.Result{Source: agent.Name()} - trimmedLine := strings.TrimRight(line, " \r\n\t") + trimmedLine := strings.TrimRight(record[0], " \r\n\t") if trimmedLine != "" { - result.Host = line + hostname, err := uncover.GetHostname(record[0]) + if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} + } + result.Host = hostname + result.Url = record[0] raw, _ := json.Marshal(result) result.Raw = raw results <- result diff --git a/uncover/agent/publicwww/request.go b/uncover/agent/publicwww/request.go index d2db63e7..be565092 100644 --- a/uncover/agent/publicwww/request.go +++ b/uncover/agent/publicwww/request.go @@ -1,7 +1,5 @@ package publicwww -import "net/url" - type Request struct { Query string `json:"query"` Start int `json:"start"` @@ -10,6 +8,6 @@ type Request struct { func (r *Request) buildURL(key string) string { return baseURL + baseEndpoint + - url.QueryEscape(`"`+r.Query+`"`) + - `/?export=urls&key=` + key + `"` + r.Query + `"` + + `/?export=csvu&key=` + key } diff --git a/uncover/result.go b/uncover/result.go index 2d70a07a..50cc31e4 100644 --- a/uncover/result.go +++ b/uncover/result.go @@ -12,6 +12,7 @@ type Result struct { IP string `json:"ip"` Port int `json:"port"` Host string `json:"host"` + Url string `json:"url"` Raw []byte `json:"-"` Error error `json:"-"` } diff --git a/uncover/util.go b/uncover/util.go index d5cfad56..3a3f9aee 100644 --- a/uncover/util.go +++ b/uncover/util.go @@ -2,6 +2,8 @@ package uncover import ( "io" + "net/url" + "github.com/projectdiscovery/retryablehttp-go" ) @@ -13,3 +15,11 @@ func NewHTTPRequest(method, url string, body io.Reader) (*retryablehttp.Request, request.Header.Set("User-Agent", "Uncover - FOSS Project (github.com/projectdiscovery/uncover)") return request, nil } + +func GetHostname(u string) (string, error) { + parsedURL, err := url.Parse(u) + if err != nil { + return "", err + } + return parsedURL.Hostname(), nil +} From e9e8559ed510b8b14f3aa778f6049970ffe1c52e Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 20 Feb 2023 23:10:40 +0100 Subject: [PATCH 23/58] tried to use http get as wip to debug with retryablehttp --- uncover/agent/publicwww/publicwww.go | 6 ++---- uncover/agent/publicwww/request.go | 6 ++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 2b54696c..bd6d9a67 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -70,11 +70,11 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan } func (agent *Agent) query(URL string, session *uncover.Session, results chan uncover.Result) []string { - resp, err := agent.queryURL(session, URL) + resp, err := http.Get(URL) if err != nil { results <- uncover.Result{Source: agent.Name(), Error: err} - return nil } + defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { @@ -124,8 +124,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon return nil, err } - request.Header.Set("Content-Type", "application/vnd.ms-excel") - agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/publicwww/request.go b/uncover/agent/publicwww/request.go index be565092..d2db63e7 100644 --- a/uncover/agent/publicwww/request.go +++ b/uncover/agent/publicwww/request.go @@ -1,5 +1,7 @@ package publicwww +import "net/url" + type Request struct { Query string `json:"query"` Start int `json:"start"` @@ -8,6 +10,6 @@ type Request struct { func (r *Request) buildURL(key string) string { return baseURL + baseEndpoint + - `"` + r.Query + `"` + - `/?export=csvu&key=` + key + url.QueryEscape(`"`+r.Query+`"`) + + `/?export=urls&key=` + key } From 1475cd4619cd95101d9bfab768780ca9de83a312 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 20 Feb 2023 23:12:16 +0100 Subject: [PATCH 24/58] commented un used code --- uncover/agent/publicwww/publicwww.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index bd6d9a67..47ee0d54 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -114,16 +114,16 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc return lines } -func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { - request, err := uncover.NewHTTPRequest( - http.MethodGet, - URL, - nil, - ) - if err != nil { - return nil, err - } - - agent.options.RateLimiter.Take() - return session.Do(request) -} +// func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { +// request, err := uncover.NewHTTPRequest( +// http.MethodGet, +// URL, +// nil, +// ) +// if err != nil { +// return nil, err +// } + +// agent.options.RateLimiter.Take() +// return session.Do(request) +// } From eced79464c717e39761989d0abafce4be2dc2caf Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Tue, 21 Feb 2023 15:52:48 +0100 Subject: [PATCH 25/58] default field for publicwww integration --- runner/runner.go | 9 +++++++-- uncover/agent/publicwww/publicwww.go | 14 -------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 85a7c4e7..dae918f2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -12,7 +12,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ratelimit" - "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/uncover/uncover" "github.com/projectdiscovery/uncover/uncover/agent/censys" "github.com/projectdiscovery/uncover/uncover/agent/criminalip" @@ -24,6 +23,7 @@ import ( "github.com/projectdiscovery/uncover/uncover/agent/shodan" "github.com/projectdiscovery/uncover/uncover/agent/shodanidb" "github.com/projectdiscovery/uncover/uncover/agent/zoomeye" + stringsutil "github.com/projectdiscovery/utils/strings" ) func init() { @@ -183,12 +183,14 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { for _, agent := range agents { wg.Add(1) go func(agent uncover.Agent, uncoverQuery *uncover.Query) { + optionFields := r.options.OutputFields defer wg.Done() keys := r.options.Provider.GetKeys() if keys.Empty() && agent.Name() != "shodan-idb" { gologger.Error().Label(agent.Name()).Msgf("empty keys\n") return } + session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout) if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) @@ -217,7 +219,10 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { "port", port, "url", result.Url, ) - outData := replacer.Replace(r.options.OutputFields) + if (result.IP == "" || port == "0") && stringsutil.ContainsAny(r.options.OutputFields, "ip", "port") { + optionFields = "host" + } + outData := replacer.Replace(optionFields) searchFor := []string{result.IP, port} if result.Host != "" || r.options.OutputFile != "" { searchFor = append(searchFor, result.Host) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 47ee0d54..a7eae88d 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -113,17 +113,3 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc return lines } - -// func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { -// request, err := uncover.NewHTTPRequest( -// http.MethodGet, -// URL, -// nil, -// ) -// if err != nil { -// return nil, err -// } - -// agent.options.RateLimiter.Take() -// return session.Do(request) -// } From 35734621a0389f0e21122a72c4bff9a4948fc025 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Feb 2023 20:38:11 +0530 Subject: [PATCH 26/58] chore(deps): bump github.com/projectdiscovery/utils from 0.0.7 to 0.0.10 (#159) * chore(deps): bump github.com/projectdiscovery/utils from 0.0.7 to 0.0.10 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.7 to 0.0.10. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.7...v0.0.10) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update go version to 1.19 * - revert go.mod version to 1.18 - remove 1.18.x from go versions in build-test.yml * update go.mod version to 1.19 * update lint-test.yml go veresion to 1.19 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ramana Reddy --- .github/workflows/build-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .goreleaser.yml | 2 +- go.mod | 4 ++-- go.sum | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ef105b22..8b042502 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-12] - go-version: [1.18.x, 1.19.x] + go-version: [1.19.x] steps: - name: Set up Go uses: actions/setup-go@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 39490315..d8e6e01b 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Checkout code uses: actions/checkout@v3 diff --git a/.goreleaser.yml b/.goreleaser.yml index f1bf8460..608fbef0 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,7 +11,7 @@ builds: - darwin goarch: - amd64 - - 386 + - '386' - arm - arm64 diff --git a/go.mod b/go.mod index 1e3cff7a..9391c649 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/uncover -go 1.18 +go 1.19 require ( github.com/hashicorp/golang-lru v0.5.4 @@ -36,7 +36,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect - github.com/projectdiscovery/utils v0.0.7 + github.com/projectdiscovery/utils v0.0.10 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index b3db6344..fd80b81d 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7 github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDkst9DA2M= -github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.10 h1:WU/sJJgE4RrXuhooKk+PKcIaTc/MT9B+ohdnK6hgpsE= +github.com/projectdiscovery/utils v0.0.10/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= From e86335235c53fe6dd4e7a30920d27f204a5b0f15 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Tue, 21 Feb 2023 23:32:04 +0100 Subject: [PATCH 27/58] started review fixes --- go.mod | 7 ++--- go.sum | 11 ++++---- uncover/agent/publicwww/publicwww.go | 42 +++++++++++++++++++--------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 46e5fcc4..683aed15 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,7 @@ require ( github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.10 - github.com/projectdiscovery/stringsutil v0.0.2 + github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/stretchr/testify v1.8.1 ) @@ -37,7 +36,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect - github.com/projectdiscovery/utils v0.0.6 + github.com/projectdiscovery/utils v0.0.7 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect @@ -48,12 +47,12 @@ require ( ) require ( - github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index aa5d420d..73b22d06 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= -github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -70,13 +68,13 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/retryablehttp-go v1.0.10 h1:A8tsJZhzpijgnvHT41JNrhiXtn8F8s7TWMC5ZxLV/Cg= -github.com/projectdiscovery/retryablehttp-go v1.0.10/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= +github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= +github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.6 h1:6SDn/5E5NxrAfcYrZ7omXPmiU9n8p0rKXZ4BAOQyzbw= -github.com/projectdiscovery/utils v0.0.6/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDkst9DA2M= +github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= @@ -112,6 +110,7 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index a7eae88d..f1ca678d 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -70,11 +70,11 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan } func (agent *Agent) query(URL string, session *uncover.Session, results chan uncover.Result) []string { - resp, err := http.Get(URL) + resp, err := agent.queryURL(session, URL) if err != nil { results <- uncover.Result{Source: agent.Name(), Error: err} + return nil } - defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { @@ -96,20 +96,36 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc } result := uncover.Result{Source: agent.Name()} - trimmedLine := strings.TrimRight(record[0], " \r\n\t") - if trimmedLine != "" { - hostname, err := uncover.GetHostname(record[0]) - if err != nil { - results <- uncover.Result{Source: agent.Name(), Error: err} + if len(record) > 0 { + trimmedLine := strings.TrimRight(record[0], " \r\n\t") + if trimmedLine != "" { + hostname, err := uncover.GetHostname(record[0]) + if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} + } + result.Host = hostname + result.Url = record[0] + raw, _ := json.Marshal(result) + result.Raw = raw + results <- result + lines = append(lines, trimmedLine) } - result.Host = hostname - result.Url = record[0] - raw, _ := json.Marshal(result) - result.Raw = raw - results <- result - lines = append(lines, trimmedLine) } } return lines } + +func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { + request, err := uncover.NewHTTPRequest( + http.MethodGet, + URL, + nil, + ) + if err != nil { + return nil, err + } + + agent.options.RateLimiter.Take() + return session.Do(request) +} From 5332d77494ab808b0cd84f2888a9770c02e8607c Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Thu, 23 Feb 2023 16:07:40 +0100 Subject: [PATCH 28/58] rate limiting logic --- runner/runner.go | 48 ++++++++++------------------------------- uncover/session.go | 26 ++++++++++++++++------ uncover/session_test.go | 3 ++- 3 files changed, 32 insertions(+), 45 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index dae918f2..a96f3eb3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -11,7 +11,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/uncover/uncover" "github.com/projectdiscovery/uncover/uncover/agent/censys" "github.com/projectdiscovery/uncover/uncover/agent/criminalip" @@ -50,31 +49,6 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return errors.New("no keys provided") } - var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter, publicwwwRatelimiter *ratelimit.Limiter - if r.options.Delay > 0 { - censysRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - fofaRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - shodanRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - shodanIdbRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - quakeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - hunterRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - zoomeyeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - netlasRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - criminalipRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - publicwwwRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - } else { - censysRateLimiter = ratelimit.NewUnlimited(context.Background()) - fofaRateLimiter = ratelimit.NewUnlimited(context.Background()) - shodanRateLimiter = ratelimit.NewUnlimited(context.Background()) - shodanIdbRateLimiter = ratelimit.NewUnlimited(context.Background()) - quakeRatelimiter = ratelimit.NewUnlimited(context.Background()) - hunterRatelimiter = ratelimit.NewUnlimited(context.Background()) - zoomeyeRatelimiter = ratelimit.NewUnlimited(context.Background()) - netlasRatelimiter = ratelimit.NewUnlimited(context.Background()) - criminalipRatelimiter = ratelimit.NewUnlimited(context.Background()) - publicwwwRatelimiter = ratelimit.NewUnlimited(context.Background()) - } - var agents []uncover.Agent if len(r.options.Shodan) > 0 { r.options.Engine = append(r.options.Engine, "shodan") @@ -125,25 +99,25 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { ) switch engine { case "shodan": - agent, err = shodan.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanRateLimiter}) + agent, err = shodan.NewWithOptions(&uncover.AgentOptions{}) case "censys": - agent, err = censys.NewWithOptions(&uncover.AgentOptions{RateLimiter: censysRateLimiter}) + agent, err = censys.NewWithOptions(&uncover.AgentOptions{}) case "fofa": - agent, err = fofa.NewWithOptions(&uncover.AgentOptions{RateLimiter: fofaRateLimiter}) + agent, err = fofa.NewWithOptions(&uncover.AgentOptions{}) case "shodan-idb": - agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanIdbRateLimiter}) + agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{}) case "quake": - agent, err = quake.NewWithOptions(&uncover.AgentOptions{RateLimiter: quakeRatelimiter}) + agent, err = quake.NewWithOptions(&uncover.AgentOptions{}) case "hunter": - agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterRatelimiter}) + agent, err = hunter.NewWithOptions(&uncover.AgentOptions{}) case "zoomeye": - agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: zoomeyeRatelimiter}) + agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{}) case "netlas": - agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: netlasRatelimiter}) + agent, err = netlas.NewWithOptions(&uncover.AgentOptions{}) case "criminalip": - agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: criminalipRatelimiter}) + agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{}) case "publicwww": - agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{RateLimiter: publicwwwRatelimiter}) + agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{}) default: err = errors.New("unknown agent type") } @@ -191,7 +165,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return } - session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout) + session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.Delay, r.options.Engine) if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) } diff --git a/uncover/session.go b/uncover/session.go index f7b3b466..710e427d 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -1,22 +1,25 @@ package uncover import ( + "context" "crypto/tls" "net/http" "net/url" "time" "github.com/pkg/errors" + "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryablehttp-go" ) type Session struct { - Keys *Keys - Client *retryablehttp.Client - RetryMax int + Keys *Keys + Client *retryablehttp.Client + RetryMax int + RateLimits map[string]*ratelimit.Limiter } -func NewSession(keys *Keys, retryMax, timeout int) (*Session, error) { +func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -37,9 +40,18 @@ func NewSession(keys *Keys, retryMax, timeout int) (*Session, error) { client := retryablehttp.NewWithHTTPClient(httpclient, options) session := &Session{ - Client: client, - Keys: keys, - RetryMax: retryMax, + Client: client, + Keys: keys, + RetryMax: retryMax, + RateLimits: make(map[string]*ratelimit.Limiter), + } + + for _, engine := range engines { + if delay > 0 { + session.RateLimits[engine] = ratelimit.New(context.Background(), 1, time.Duration(delay)) + } else { + session.RateLimits[engine] = ratelimit.NewUnlimited(context.Background()) + } } return session, nil diff --git a/uncover/session_test.go b/uncover/session_test.go index c03caa42..554d6692 100644 --- a/uncover/session_test.go +++ b/uncover/session_test.go @@ -18,7 +18,8 @@ func TestSessionRetry(t *testing.T) { t.Log("Slept for 10 seconds") })) ts := httptest.NewServer(router) - session, err := NewSession(&Keys{}, 5, 3) + engines := []string{"shoda,", "publicwww"} + session, err := NewSession(&Keys{}, 5, 3, 60, engines) require.Nil(t, err) req, err := retryablehttp.NewRequest(http.MethodGet, ts.URL, nil) require.Nil(t, err) From 4ed60d123e8802d284cee9e87c20a4c5cae2ab02 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Thu, 23 Feb 2023 23:16:34 +0100 Subject: [PATCH 29/58] code refactoring on session management --- runner/runner.go | 21 ++++++++++--------- uncover/agent.go | 6 ------ uncover/agent/censys/censys.go | 10 ++------- uncover/agent/criminalip/criminalip.go | 18 ++++++----------- uncover/agent/fofa/fofa.go | 10 ++------- uncover/agent/hunter/hunter.go | 10 ++------- uncover/agent/netlas/netlas.go | 10 ++------- uncover/agent/publicwww/publicwww.go | 10 ++------- uncover/agent/quake/quake.go | 10 ++------- uncover/agent/shodan/shodan.go | 10 ++------- uncover/agent/shodanidb/shodan.go | 10 ++------- uncover/agent/zoomeye/zoomeye.go | 10 ++------- uncover/session.go | 28 ++++++++++++++++++-------- 13 files changed, 55 insertions(+), 108 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index a96f3eb3..cdfea326 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -99,25 +99,25 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { ) switch engine { case "shodan": - agent, err = shodan.NewWithOptions(&uncover.AgentOptions{}) + agent, err = shodan.New() case "censys": - agent, err = censys.NewWithOptions(&uncover.AgentOptions{}) + agent, err = censys.New() case "fofa": - agent, err = fofa.NewWithOptions(&uncover.AgentOptions{}) + agent, err = fofa.New() case "shodan-idb": - agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{}) + agent, err = shodanidb.New() case "quake": - agent, err = quake.NewWithOptions(&uncover.AgentOptions{}) + agent, err = quake.New() case "hunter": - agent, err = hunter.NewWithOptions(&uncover.AgentOptions{}) + agent, err = hunter.New() case "zoomeye": - agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{}) + agent, err = zoomeye.New() case "netlas": - agent, err = netlas.NewWithOptions(&uncover.AgentOptions{}) + agent, err = netlas.New() case "criminalip": - agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{}) + agent, err = criminalip.New() case "publicwww": - agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{}) + agent, err = publicwww.New() default: err = errors.New("unknown agent type") } @@ -169,6 +169,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) } + ch, err := agent.Query(session, uncoverQuery) if err != nil { gologger.Warning().Msgf("%s\n", err) diff --git a/uncover/agent.go b/uncover/agent.go index 0fe6c44e..9e1b09b1 100644 --- a/uncover/agent.go +++ b/uncover/agent.go @@ -1,11 +1,5 @@ package uncover -import "github.com/projectdiscovery/ratelimit" - -type AgentOptions struct { - RateLimiter *ratelimit.Limiter -} - type Query struct { Query string Limit int diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 1da386cf..557e85b0 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -16,18 +16,12 @@ const ( MaxPerPage = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "censys" } @@ -75,7 +69,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest } request.Header.Set("Accept", "application/json") request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index 3c3dd14a..78b08b01 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -15,18 +15,12 @@ const ( URL = "https://api.criminalip.io/v1/banner/search?query=%s&offset=%d" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "criminalip" } @@ -44,8 +38,8 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan currentPage := 1 for { criminalipRequest := &CriminalIPRequest{ - Query: query.Query, - Offset: currentPage, + Query: query.Query, + Offset: currentPage, } criminalipResponse := agent.query(URL, session, criminalipRequest, results) @@ -73,7 +67,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, criminalipReq return nil, err } request.Header.Set("x-api-key", session.Keys.CriminalIPToken) - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } @@ -106,6 +100,6 @@ func (agent *Agent) query(URL string, session *uncover.Session, criminalipReques } type CriminalIPRequest struct { - Query string - Offset int + Query string + Offset int } diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 3344cf0f..9fa5a8be 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -18,18 +18,12 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "fofa" } @@ -77,7 +71,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest * return nil, err } request.Header.Set("Accept", "application/json") - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index e0ef76f6..69aa937a 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -15,18 +15,12 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "hunter" } @@ -103,6 +97,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, hunterRequest return nil, err } request.Header.Set("Accept", "application/json") - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index ac2337c4..4af86317 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -14,18 +14,12 @@ const ( contentType = "application/json" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "netlas" } @@ -103,6 +97,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon request.Header.Set("Content-Type", contentType) request.Header.Set("X-API-Key", session.Keys.NetlasToken) - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index f1ca678d..b5a372f9 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -16,18 +16,12 @@ const ( baseEndpoint = "websites/" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "publicwww" } @@ -126,6 +120,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon return nil, err } - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 7c3b2047..0c4e4cc1 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -14,18 +14,12 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "quake" } @@ -110,6 +104,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, quakeRequest request.Header.Set("Content-Type", "application/json") request.Header.Set("X-QuakeToken", session.Keys.QuakeToken) - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index d6c9e678..e6353426 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -15,18 +15,12 @@ const ( URL = "https://api.shodan.io/shodan/host/search?key=%s&query=%s&page=%d" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "shodan" } @@ -74,7 +68,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index 305e15a6..b93b47d1 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -16,18 +16,12 @@ const ( URL = "https://internetdb.shodan.io/%s" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "shodan-idb" } @@ -55,7 +49,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index 0e6270f8..891e714c 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -16,18 +16,12 @@ const ( URL = "https://api.zoomeye.org/host/search?query=%s&page=%d" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "zoomeye" } @@ -77,7 +71,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, zoomeyeReques return nil, err } request.Header.Set("API-KEY", session.Keys.ZoomEyeToken) - agent.options.RateLimiter.Take() + session.RateLimits.Take(agent.Name()) return session.Do(request) } diff --git a/uncover/session.go b/uncover/session.go index 710e427d..21dd43a0 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -16,7 +16,7 @@ type Session struct { Keys *Keys Client *retryablehttp.Client RetryMax int - RateLimits map[string]*ratelimit.Limiter + RateLimits *ratelimit.MultiLimiter } func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Session, error) { @@ -43,15 +43,27 @@ func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Se Client: client, Keys: keys, RetryMax: retryMax, - RateLimits: make(map[string]*ratelimit.Limiter), + RateLimits: &ratelimit.MultiLimiter{}, } - for _, engine := range engines { - if delay > 0 { - session.RateLimits[engine] = ratelimit.New(context.Background(), 1, time.Duration(delay)) - } else { - session.RateLimits[engine] = ratelimit.NewUnlimited(context.Background()) - } + var err error + rateLimitOpts := &ratelimit.Options{ + MaxCount: uint(retryMax), + Duration: time.Duration(delay), + IsUnlimited: delay == 0, + } + + rateLimitOpts.Key = engines[0] + + session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), rateLimitOpts) + if err != nil { + return &Session{}, nil + } + + for _, engine := range engines[1:] { + engineOpts := rateLimitOpts + engineOpts.Key = engine + session.RateLimits.Add(engineOpts) } return session, nil From 164b41d8c6727898700936340196d2995e614fa2 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Thu, 23 Feb 2023 23:22:01 +0100 Subject: [PATCH 30/58] error management --- uncover/agent/censys/censys.go | 5 ++++- uncover/agent/criminalip/criminalip.go | 5 ++++- uncover/agent/fofa/fofa.go | 5 ++++- uncover/agent/hunter/hunter.go | 5 ++++- uncover/agent/netlas/netlas.go | 5 ++++- uncover/agent/publicwww/publicwww.go | 5 ++++- uncover/agent/quake/quake.go | 5 ++++- uncover/agent/shodan/shodan.go | 5 ++++- uncover/agent/shodanidb/shodan.go | 5 ++++- uncover/agent/zoomeye/zoomeye.go | 5 ++++- uncover/session.go | 5 ++++- 11 files changed, 44 insertions(+), 11 deletions(-) diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 557e85b0..720897bb 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -69,7 +69,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest } request.Header.Set("Accept", "application/json") request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index 78b08b01..a81a64b3 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -67,7 +67,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, criminalipReq return nil, err } request.Header.Set("x-api-key", session.Keys.CriminalIPToken) - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 9fa5a8be..208e1ad5 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -71,7 +71,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest * return nil, err } request.Header.Set("Accept", "application/json") - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index 69aa937a..a80577c9 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -97,6 +97,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, hunterRequest return nil, err } request.Header.Set("Accept", "application/json") - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index 4af86317..d08eb64c 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -97,6 +97,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon request.Header.Set("Content-Type", contentType) request.Header.Set("X-API-Key", session.Keys.NetlasToken) - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index b5a372f9..7c7838e0 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -120,6 +120,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon return nil, err } - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 0c4e4cc1..76cb5cde 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -104,6 +104,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, quakeRequest request.Header.Set("Content-Type", "application/json") request.Header.Set("X-QuakeToken", session.Keys.QuakeToken) - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index e6353426..b808b192 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -68,7 +68,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index b93b47d1..29a61ff9 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -49,7 +49,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index 891e714c..a630ea26 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -71,7 +71,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, zoomeyeReques return nil, err } request.Header.Set("API-KEY", session.Keys.ZoomEyeToken) - session.RateLimits.Take(agent.Name()) + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/session.go b/uncover/session.go index 21dd43a0..48f606a5 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -63,7 +63,10 @@ func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Se for _, engine := range engines[1:] { engineOpts := rateLimitOpts engineOpts.Key = engine - session.RateLimits.Add(engineOpts) + err := session.RateLimits.Add(engineOpts) + if err != nil { + return nil, err + } } return session, nil From cc0e00ba69e158a330bc356c4c08e214f6b4bb88 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Thu, 23 Feb 2023 23:25:17 +0100 Subject: [PATCH 31/58] go mod - sum --- go.mod | 3 +-- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index e658837a..0d7ee54a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/retryablehttp-go v1.0.11 - github.com/projectdiscovery/stringsutil v0.0.2 github.com/stretchr/testify v1.8.1 ) @@ -47,12 +46,12 @@ require ( ) require ( + github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect - github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index 87e0b073..bbd7426d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -70,8 +72,6 @@ github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQL github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo= github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= -github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/utils v0.0.10 h1:WU/sJJgE4RrXuhooKk+PKcIaTc/MT9B+ohdnK6hgpsE= github.com/projectdiscovery/utils v0.0.10/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -108,9 +108,7 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -122,17 +120,13 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From d593f6615b0047179364fd0c1b9e64e408593d69 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Tue, 28 Feb 2023 19:36:41 +0100 Subject: [PATCH 32/58] restored logic - session and rate limit logic moved to https://github.com/projectdiscovery/uncover/pull/165 --- runner/runner.go | 49 +++++++++++++++++++------- uncover/agent.go | 6 ++++ uncover/agent/censys/censys.go | 13 ++++--- uncover/agent/criminalip/criminalip.go | 13 ++++--- uncover/agent/fofa/fofa.go | 13 ++++--- uncover/agent/hunter/hunter.go | 13 ++++--- uncover/agent/netlas/netlas.go | 13 ++++--- uncover/agent/publicwww/publicwww.go | 13 ++++--- uncover/agent/quake/quake.go | 13 ++++--- uncover/agent/shodan/shodan.go | 13 ++++--- uncover/agent/shodanidb/shodan.go | 13 ++++--- uncover/agent/zoomeye/zoomeye.go | 13 ++++--- uncover/session.go | 41 ++++----------------- uncover/session_test.go | 3 +- 14 files changed, 131 insertions(+), 98 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index cdfea326..dae918f2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/uncover/uncover" "github.com/projectdiscovery/uncover/uncover/agent/censys" "github.com/projectdiscovery/uncover/uncover/agent/criminalip" @@ -49,6 +50,31 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return errors.New("no keys provided") } + var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter, publicwwwRatelimiter *ratelimit.Limiter + if r.options.Delay > 0 { + censysRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + fofaRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + shodanRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + shodanIdbRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + quakeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + hunterRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + zoomeyeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + netlasRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + criminalipRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + publicwwwRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + } else { + censysRateLimiter = ratelimit.NewUnlimited(context.Background()) + fofaRateLimiter = ratelimit.NewUnlimited(context.Background()) + shodanRateLimiter = ratelimit.NewUnlimited(context.Background()) + shodanIdbRateLimiter = ratelimit.NewUnlimited(context.Background()) + quakeRatelimiter = ratelimit.NewUnlimited(context.Background()) + hunterRatelimiter = ratelimit.NewUnlimited(context.Background()) + zoomeyeRatelimiter = ratelimit.NewUnlimited(context.Background()) + netlasRatelimiter = ratelimit.NewUnlimited(context.Background()) + criminalipRatelimiter = ratelimit.NewUnlimited(context.Background()) + publicwwwRatelimiter = ratelimit.NewUnlimited(context.Background()) + } + var agents []uncover.Agent if len(r.options.Shodan) > 0 { r.options.Engine = append(r.options.Engine, "shodan") @@ -99,25 +125,25 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { ) switch engine { case "shodan": - agent, err = shodan.New() + agent, err = shodan.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanRateLimiter}) case "censys": - agent, err = censys.New() + agent, err = censys.NewWithOptions(&uncover.AgentOptions{RateLimiter: censysRateLimiter}) case "fofa": - agent, err = fofa.New() + agent, err = fofa.NewWithOptions(&uncover.AgentOptions{RateLimiter: fofaRateLimiter}) case "shodan-idb": - agent, err = shodanidb.New() + agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanIdbRateLimiter}) case "quake": - agent, err = quake.New() + agent, err = quake.NewWithOptions(&uncover.AgentOptions{RateLimiter: quakeRatelimiter}) case "hunter": - agent, err = hunter.New() + agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterRatelimiter}) case "zoomeye": - agent, err = zoomeye.New() + agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: zoomeyeRatelimiter}) case "netlas": - agent, err = netlas.New() + agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: netlasRatelimiter}) case "criminalip": - agent, err = criminalip.New() + agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: criminalipRatelimiter}) case "publicwww": - agent, err = publicwww.New() + agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{RateLimiter: publicwwwRatelimiter}) default: err = errors.New("unknown agent type") } @@ -165,11 +191,10 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return } - session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.Delay, r.options.Engine) + session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout) if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) } - ch, err := agent.Query(session, uncoverQuery) if err != nil { gologger.Warning().Msgf("%s\n", err) diff --git a/uncover/agent.go b/uncover/agent.go index 9e1b09b1..0fe6c44e 100644 --- a/uncover/agent.go +++ b/uncover/agent.go @@ -1,5 +1,11 @@ package uncover +import "github.com/projectdiscovery/ratelimit" + +type AgentOptions struct { + RateLimiter *ratelimit.Limiter +} + type Query struct { Query string Limit int diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 720897bb..1da386cf 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -16,12 +16,18 @@ const ( MaxPerPage = 100 ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "censys" } @@ -69,10 +75,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest } request.Header.Set("Accept", "application/json") request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index a81a64b3..e57af357 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -15,12 +15,18 @@ const ( URL = "https://api.criminalip.io/v1/banner/search?query=%s&offset=%d" ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "criminalip" } @@ -67,10 +73,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, criminalipReq return nil, err } request.Header.Set("x-api-key", session.Keys.CriminalIPToken) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 208e1ad5..3344cf0f 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -18,12 +18,18 @@ const ( Size = 100 ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "fofa" } @@ -71,10 +77,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest * return nil, err } request.Header.Set("Accept", "application/json") - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index a80577c9..e0ef76f6 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -15,12 +15,18 @@ const ( Size = 100 ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "hunter" } @@ -97,9 +103,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, hunterRequest return nil, err } request.Header.Set("Accept", "application/json") - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index d08eb64c..ac2337c4 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -14,12 +14,18 @@ const ( contentType = "application/json" ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "netlas" } @@ -97,9 +103,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon request.Header.Set("Content-Type", contentType) request.Header.Set("X-API-Key", session.Keys.NetlasToken) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 7c7838e0..f1ca678d 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -16,12 +16,18 @@ const ( baseEndpoint = "websites/" ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "publicwww" } @@ -120,9 +126,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon return nil, err } - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 76cb5cde..7c3b2047 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -14,12 +14,18 @@ const ( Size = 100 ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "quake" } @@ -104,9 +110,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, quakeRequest request.Header.Set("Content-Type", "application/json") request.Header.Set("X-QuakeToken", session.Keys.QuakeToken) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index b808b192..d6c9e678 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -15,12 +15,18 @@ const ( URL = "https://api.shodan.io/shodan/host/search?key=%s&query=%s&page=%d" ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "shodan" } @@ -68,10 +74,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index 29a61ff9..305e15a6 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -16,12 +16,18 @@ const ( URL = "https://internetdb.shodan.io/%s" ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "shodan-idb" } @@ -49,10 +55,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index a630ea26..0e6270f8 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -16,12 +16,18 @@ const ( URL = "https://api.zoomeye.org/host/search?query=%s&page=%d" ) -type Agent struct{} +type Agent struct { + options *uncover.AgentOptions +} func New() (uncover.Agent, error) { return &Agent{}, nil } +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + func (agent *Agent) Name() string { return "zoomeye" } @@ -71,10 +77,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, zoomeyeReques return nil, err } request.Header.Set("API-KEY", session.Keys.ZoomEyeToken) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } + agent.options.RateLimiter.Take() return session.Do(request) } diff --git a/uncover/session.go b/uncover/session.go index 48f606a5..f7b3b466 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -1,25 +1,22 @@ package uncover import ( - "context" "crypto/tls" "net/http" "net/url" "time" "github.com/pkg/errors" - "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryablehttp-go" ) type Session struct { - Keys *Keys - Client *retryablehttp.Client - RetryMax int - RateLimits *ratelimit.MultiLimiter + Keys *Keys + Client *retryablehttp.Client + RetryMax int } -func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Session, error) { +func NewSession(keys *Keys, retryMax, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -40,33 +37,9 @@ func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Se client := retryablehttp.NewWithHTTPClient(httpclient, options) session := &Session{ - Client: client, - Keys: keys, - RetryMax: retryMax, - RateLimits: &ratelimit.MultiLimiter{}, - } - - var err error - rateLimitOpts := &ratelimit.Options{ - MaxCount: uint(retryMax), - Duration: time.Duration(delay), - IsUnlimited: delay == 0, - } - - rateLimitOpts.Key = engines[0] - - session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), rateLimitOpts) - if err != nil { - return &Session{}, nil - } - - for _, engine := range engines[1:] { - engineOpts := rateLimitOpts - engineOpts.Key = engine - err := session.RateLimits.Add(engineOpts) - if err != nil { - return nil, err - } + Client: client, + Keys: keys, + RetryMax: retryMax, } return session, nil diff --git a/uncover/session_test.go b/uncover/session_test.go index 554d6692..c03caa42 100644 --- a/uncover/session_test.go +++ b/uncover/session_test.go @@ -18,8 +18,7 @@ func TestSessionRetry(t *testing.T) { t.Log("Slept for 10 seconds") })) ts := httptest.NewServer(router) - engines := []string{"shoda,", "publicwww"} - session, err := NewSession(&Keys{}, 5, 3, 60, engines) + session, err := NewSession(&Keys{}, 5, 3) require.Nil(t, err) req, err := retryablehttp.NewRequest(http.MethodGet, ts.URL, nil) require.Nil(t, err) From 71fd9645d5e01369fd3db8bfd180556c279f396f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 1 Mar 2023 08:38:08 +0100 Subject: [PATCH 33/58] continue iteration on err --- uncover/agent/publicwww/publicwww.go | 1 + 1 file changed, 1 insertion(+) diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index f1ca678d..94a3d2c8 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -102,6 +102,7 @@ func (agent *Agent) query(URL string, session *uncover.Session, results chan unc hostname, err := uncover.GetHostname(record[0]) if err != nil { results <- uncover.Result{Source: agent.Name(), Error: err} + continue } result.Host = hostname result.Url = record[0] From 9fc13e7b1c51f06653b6b0527513c4893828d356 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 07:47:20 +0000 Subject: [PATCH 34/58] chore(deps): bump github.com/projectdiscovery/utils Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.10 to 0.0.13. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.10...v0.0.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 0d7ee54a..3600dd16 100644 --- a/go.mod +++ b/go.mod @@ -35,13 +35,13 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect - github.com/projectdiscovery/utils v0.0.10 + github.com/projectdiscovery/utils v0.0.13 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sys v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index bbd7426d..0b2fdb75 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQL github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo= github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= -github.com/projectdiscovery/utils v0.0.10 h1:WU/sJJgE4RrXuhooKk+PKcIaTc/MT9B+ohdnK6hgpsE= -github.com/projectdiscovery/utils v0.0.10/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw= +github.com/projectdiscovery/utils v0.0.13 h1:Bvguo7avXN+Zs9tHL+8+Qld65nwIE8Kr9qddErDDqmw= +github.com/projectdiscovery/utils v0.0.13/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= @@ -109,8 +109,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -121,14 +121,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= From a43006762c0eee0e78c001f4f83401758066ec6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 07:47:21 +0000 Subject: [PATCH 35/58] chore(deps): bump github.com/stretchr/testify from 1.8.1 to 1.8.2 Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.1 to 1.8.2. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.1...v1.8.2) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0d7ee54a..cb78e765 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/retryablehttp-go v1.0.11 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 ) require ( diff --git a/go.sum b/go.sum index bbd7426d..3473ecd0 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= From 28a8ab4c4470c2d94ccb5b1badaef75fb307a547 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 07:53:11 +0000 Subject: [PATCH 36/58] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.11 to 1.0.12. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.11...v1.0.12) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e992b6ac..c4353c7a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.11 + github.com/projectdiscovery/retryablehttp-go v1.0.12 github.com/stretchr/testify v1.8.2 ) diff --git a/go.sum b/go.sum index cf1e6c52..26be39b6 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oT github.com/projectdiscovery/mapcidr v1.1.0/go.mod h1:hck0bWXka5ZkUaBG+TWt99bzLy+4hAg9oANhEmm3GNs= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo= -github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= +github.com/projectdiscovery/retryablehttp-go v1.0.12 h1:kjeXJ4V6ZwgoMeKV2dINERjxAsBPB/p5+NV2aSQbrrU= +github.com/projectdiscovery/retryablehttp-go v1.0.12/go.mod h1:dnJK347etdVKz7ljhemUhBK6EpDnVf0U1O8dGj7MM+0= github.com/projectdiscovery/utils v0.0.13 h1:Bvguo7avXN+Zs9tHL+8+Qld65nwIE8Kr9qddErDDqmw= github.com/projectdiscovery/utils v0.0.13/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= From 6570ce61130bcfde64f189816f5f5eb6e28160a4 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 1 Mar 2023 20:25:22 +0100 Subject: [PATCH 37/58] started refactoring --- runner/runner.go | 49 +++++++------------------- uncover/agent.go | 6 ---- uncover/agent/censys/censys.go | 13 +++---- uncover/agent/criminalip/criminalip.go | 13 +++---- uncover/agent/fofa/fofa.go | 13 +++---- uncover/agent/hunter/hunter.go | 13 +++---- uncover/agent/netlas/netlas.go | 13 +++---- uncover/agent/publicwww/publicwww.go | 13 +++---- uncover/agent/quake/quake.go | 13 +++---- uncover/agent/shodan/shodan.go | 13 +++---- uncover/agent/shodanidb/shodan.go | 13 +++---- uncover/agent/zoomeye/zoomeye.go | 13 +++---- uncover/session.go | 40 +++++++++++++++++---- uncover/session_test.go | 3 +- 14 files changed, 97 insertions(+), 131 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index dae918f2..cdfea326 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -11,7 +11,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/uncover/uncover" "github.com/projectdiscovery/uncover/uncover/agent/censys" "github.com/projectdiscovery/uncover/uncover/agent/criminalip" @@ -50,31 +49,6 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return errors.New("no keys provided") } - var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter, publicwwwRatelimiter *ratelimit.Limiter - if r.options.Delay > 0 { - censysRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - fofaRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - shodanRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - shodanIdbRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - quakeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - hunterRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - zoomeyeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - netlasRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - criminalipRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - publicwwwRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) - } else { - censysRateLimiter = ratelimit.NewUnlimited(context.Background()) - fofaRateLimiter = ratelimit.NewUnlimited(context.Background()) - shodanRateLimiter = ratelimit.NewUnlimited(context.Background()) - shodanIdbRateLimiter = ratelimit.NewUnlimited(context.Background()) - quakeRatelimiter = ratelimit.NewUnlimited(context.Background()) - hunterRatelimiter = ratelimit.NewUnlimited(context.Background()) - zoomeyeRatelimiter = ratelimit.NewUnlimited(context.Background()) - netlasRatelimiter = ratelimit.NewUnlimited(context.Background()) - criminalipRatelimiter = ratelimit.NewUnlimited(context.Background()) - publicwwwRatelimiter = ratelimit.NewUnlimited(context.Background()) - } - var agents []uncover.Agent if len(r.options.Shodan) > 0 { r.options.Engine = append(r.options.Engine, "shodan") @@ -125,25 +99,25 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { ) switch engine { case "shodan": - agent, err = shodan.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanRateLimiter}) + agent, err = shodan.New() case "censys": - agent, err = censys.NewWithOptions(&uncover.AgentOptions{RateLimiter: censysRateLimiter}) + agent, err = censys.New() case "fofa": - agent, err = fofa.NewWithOptions(&uncover.AgentOptions{RateLimiter: fofaRateLimiter}) + agent, err = fofa.New() case "shodan-idb": - agent, err = shodanidb.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanIdbRateLimiter}) + agent, err = shodanidb.New() case "quake": - agent, err = quake.NewWithOptions(&uncover.AgentOptions{RateLimiter: quakeRatelimiter}) + agent, err = quake.New() case "hunter": - agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterRatelimiter}) + agent, err = hunter.New() case "zoomeye": - agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: zoomeyeRatelimiter}) + agent, err = zoomeye.New() case "netlas": - agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: netlasRatelimiter}) + agent, err = netlas.New() case "criminalip": - agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: criminalipRatelimiter}) + agent, err = criminalip.New() case "publicwww": - agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{RateLimiter: publicwwwRatelimiter}) + agent, err = publicwww.New() default: err = errors.New("unknown agent type") } @@ -191,10 +165,11 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return } - session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout) + session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.Delay, r.options.Engine) if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) } + ch, err := agent.Query(session, uncoverQuery) if err != nil { gologger.Warning().Msgf("%s\n", err) diff --git a/uncover/agent.go b/uncover/agent.go index 0fe6c44e..9e1b09b1 100644 --- a/uncover/agent.go +++ b/uncover/agent.go @@ -1,11 +1,5 @@ package uncover -import "github.com/projectdiscovery/ratelimit" - -type AgentOptions struct { - RateLimiter *ratelimit.Limiter -} - type Query struct { Query string Limit int diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 1da386cf..720897bb 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -16,18 +16,12 @@ const ( MaxPerPage = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "censys" } @@ -75,7 +69,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest } request.Header.Set("Accept", "application/json") request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index e57af357..a81a64b3 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -15,18 +15,12 @@ const ( URL = "https://api.criminalip.io/v1/banner/search?query=%s&offset=%d" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "criminalip" } @@ -73,7 +67,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, criminalipReq return nil, err } request.Header.Set("x-api-key", session.Keys.CriminalIPToken) - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 3344cf0f..208e1ad5 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -18,18 +18,12 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "fofa" } @@ -77,7 +71,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest * return nil, err } request.Header.Set("Accept", "application/json") - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index e0ef76f6..a80577c9 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -15,18 +15,12 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "hunter" } @@ -103,6 +97,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, hunterRequest return nil, err } request.Header.Set("Accept", "application/json") - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index ac2337c4..d08eb64c 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -14,18 +14,12 @@ const ( contentType = "application/json" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "netlas" } @@ -103,6 +97,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon request.Header.Set("Content-Type", contentType) request.Header.Set("X-API-Key", session.Keys.NetlasToken) - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 94a3d2c8..3578fe43 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -16,18 +16,12 @@ const ( baseEndpoint = "websites/" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "publicwww" } @@ -127,6 +121,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon return nil, err } - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 7c3b2047..76cb5cde 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -14,18 +14,12 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "quake" } @@ -110,6 +104,9 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, quakeRequest request.Header.Set("Content-Type", "application/json") request.Header.Set("X-QuakeToken", session.Keys.QuakeToken) - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index d6c9e678..b808b192 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -15,18 +15,12 @@ const ( URL = "https://api.shodan.io/shodan/host/search?key=%s&query=%s&page=%d" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "shodan" } @@ -74,7 +68,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index 305e15a6..29a61ff9 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -16,18 +16,12 @@ const ( URL = "https://internetdb.shodan.io/%s" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "shodan-idb" } @@ -55,7 +49,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index 0e6270f8..a630ea26 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -16,18 +16,12 @@ const ( URL = "https://api.zoomeye.org/host/search?query=%s&page=%d" ) -type Agent struct { - options *uncover.AgentOptions -} +type Agent struct{} func New() (uncover.Agent, error) { return &Agent{}, nil } -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} - func (agent *Agent) Name() string { return "zoomeye" } @@ -77,7 +71,10 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, zoomeyeReques return nil, err } request.Header.Set("API-KEY", session.Keys.ZoomEyeToken) - agent.options.RateLimiter.Take() + err = session.RateLimits.Take(agent.Name()) + if err != nil { + return nil, err + } return session.Do(request) } diff --git a/uncover/session.go b/uncover/session.go index f7b3b466..fcf0ac25 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -1,22 +1,25 @@ package uncover import ( + "context" "crypto/tls" "net/http" "net/url" "time" "github.com/pkg/errors" + "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryablehttp-go" ) type Session struct { - Keys *Keys - Client *retryablehttp.Client - RetryMax int + Keys *Keys + Client *retryablehttp.Client + RetryMax int + RateLimits *ratelimit.MultiLimiter } -func NewSession(keys *Keys, retryMax, timeout int) (*Session, error) { +func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -37,9 +40,32 @@ func NewSession(keys *Keys, retryMax, timeout int) (*Session, error) { client := retryablehttp.NewWithHTTPClient(httpclient, options) session := &Session{ - Client: client, - Keys: keys, - RetryMax: retryMax, + Client: client, + Keys: keys, + RetryMax: retryMax, + RateLimits: &ratelimit.MultiLimiter{}, + } + + var err error + rateLimitOpts := &ratelimit.Options{ + MaxCount: 1, + Duration: time.Duration(delay), + IsUnlimited: delay == 0, + } + + rateLimitOpts.Key = engines[0] + session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), rateLimitOpts) + if err != nil { + return &Session{}, nil + } + + for _, engine := range engines[1:] { + engineOpts := rateLimitOpts + engineOpts.Key = engine + err := session.RateLimits.Add(engineOpts) + if err != nil { + return nil, err + } } return session, nil diff --git a/uncover/session_test.go b/uncover/session_test.go index c03caa42..554d6692 100644 --- a/uncover/session_test.go +++ b/uncover/session_test.go @@ -18,7 +18,8 @@ func TestSessionRetry(t *testing.T) { t.Log("Slept for 10 seconds") })) ts := httptest.NewServer(router) - session, err := NewSession(&Keys{}, 5, 3) + engines := []string{"shoda,", "publicwww"} + session, err := NewSession(&Keys{}, 5, 3, 60, engines) require.Nil(t, err) req, err := retryablehttp.NewRequest(http.MethodGet, ts.URL, nil) require.Nil(t, err) From f0e092270c4d055135ce21be8a9ca8c3850620ed Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 1 Mar 2023 20:32:43 +0100 Subject: [PATCH 38/58] removed New() replaced New() with agents = append(agents, &{PPROVIDER-NAME}.Agent{}) --- runner/runner.go | 25 +++++++++++-------------- uncover/agent/censys/censys.go | 4 ---- uncover/agent/criminalip/criminalip.go | 4 ---- uncover/agent/fofa/fofa.go | 4 ---- uncover/agent/hunter/hunter.go | 4 ---- uncover/agent/netlas/netlas.go | 4 ---- uncover/agent/publicwww/publicwww.go | 4 ---- uncover/agent/quake/quake.go | 4 ---- uncover/agent/shodan/shodan.go | 4 ---- uncover/agent/shodanidb/shodan.go | 4 ---- uncover/agent/zoomeye/zoomeye.go | 4 ---- uncover/session.go | 8 +++++++- 12 files changed, 18 insertions(+), 55 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index cdfea326..a3422369 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -17,7 +17,6 @@ import ( "github.com/projectdiscovery/uncover/uncover/agent/fofa" "github.com/projectdiscovery/uncover/uncover/agent/hunter" "github.com/projectdiscovery/uncover/uncover/agent/netlas" - "github.com/projectdiscovery/uncover/uncover/agent/publicwww" "github.com/projectdiscovery/uncover/uncover/agent/quake" "github.com/projectdiscovery/uncover/uncover/agent/shodan" "github.com/projectdiscovery/uncover/uncover/agent/shodanidb" @@ -94,37 +93,35 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { // declare clients for _, engine := range r.options.Engine { var ( - agent uncover.Agent - err error + err error ) switch engine { case "shodan": - agent, err = shodan.New() + agents = append(agents, &shodan.Agent{}) case "censys": - agent, err = censys.New() + agents = append(agents, &censys.Agent{}) case "fofa": - agent, err = fofa.New() + agents = append(agents, &fofa.Agent{}) case "shodan-idb": - agent, err = shodanidb.New() + agents = append(agents, &shodanidb.Agent{}) case "quake": - agent, err = quake.New() + agents = append(agents, &quake.Agent{}) case "hunter": - agent, err = hunter.New() + agents = append(agents, &hunter.Agent{}) case "zoomeye": - agent, err = zoomeye.New() + agents = append(agents, &zoomeye.Agent{}) case "netlas": - agent, err = netlas.New() + agents = append(agents, &netlas.Agent{}) case "criminalip": - agent, err = criminalip.New() + agents = append(agents, &criminalip.Agent{}) case "publicwww": - agent, err = publicwww.New() + agents = append(agents, &criminalip.Agent{}) default: err = errors.New("unknown agent type") } if err != nil { return err } - agents = append(agents, agent) } // open the output file - always overwrite diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 720897bb..9acb988f 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -18,10 +18,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "censys" } diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index a81a64b3..e4384782 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -17,10 +17,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "criminalip" } diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 208e1ad5..1bbd60bd 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -20,10 +20,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "fofa" } diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index a80577c9..773ea328 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -17,10 +17,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "hunter" } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index d08eb64c..9ca0cbf8 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -16,10 +16,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "netlas" } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 3578fe43..88158fb8 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -18,10 +18,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "publicwww" } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 76cb5cde..39f2c694 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -16,10 +16,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "quake" } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index b808b192..6dd0ea6d 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -17,10 +17,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "shodan" } diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index 29a61ff9..16c0e2e8 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -18,10 +18,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "shodan-idb" } diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index a630ea26..288d0815 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -18,10 +18,6 @@ const ( type Agent struct{} -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - func (agent *Agent) Name() string { return "zoomeye" } diff --git a/uncover/session.go b/uncover/session.go index fcf0ac25..55b098fa 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -71,7 +71,13 @@ func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Se return session, nil } -func (s *Session) Do(request *retryablehttp.Request) (*http.Response, error) { +func (s *Session) Do(request *retryablehttp.Request, sources ...string) (*http.Response, error) { + for _, source := range sources { + err := s.RateLimits.Take(source) + if err != nil { + return nil, err + } + } // close request connection (does not reuse connections) request.Close = true resp, err := s.Client.Do(request) From 0d1c7880a8d2f829843a93b9c39a965274eebe9b Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 1 Mar 2023 20:58:06 +0100 Subject: [PATCH 39/58] logic as Subfinder, using MultiLimiter --- README.md | 11 ++++++++--- runner/options.go | 12 ++---------- runner/runner.go | 2 +- uncover/session.go | 6 +++--- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 731aab7d..c81d68c9 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Usage: Flags: INPUT: -q, -query string[] search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt') - -e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan) + -e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip, publicwww) (default shodan) SEARCH-ENGINE: -s, -shodan string[] search query for shodan (example: -shodan 'query.txt') @@ -83,12 +83,13 @@ SEARCH-ENGINE: -ze, -zoomeye string[] search query for zoomeye (example: -zoomeye 'query.txt') -ne, -netlas string[] search query for netlas (example: -netlas 'query.txt') -cl, -criminalip string[] search query for criminalip (example: -criminalip 'query.txt') + -pw, -publicwww string[] search query for publicwww (example: -publicwww 'query.txt') CONFIG: -pc, -provider string provider configuration file (default "$HOME/.config/uncover/provider-config.yaml") -config string flag configuration file (default "$HOME/.config/uncover/config.yaml") -timeout int timeout in seconds (default 30) - -delay int delay between requests in seconds (0 to disable) (default 1) + -rl, -rate-limit int maximum number of http requests to send per second -retry int number of times to retry a failed request (default 2) OUTPUT: @@ -137,6 +138,9 @@ netlas: criminalip: - CRIMINALIP_API_KEY_1 - CRIMINALIP_API_KEY_2 +publicwww: + - PUBLICWWW_API_KEY_1 + - PUBLICWWW_API_KEY_2 ``` When multiple keys/credentials are specified for same provider in the config file, random key will be used for each execution. @@ -154,9 +158,10 @@ export HUNTER_API_KEY=xxx export ZOOMEYE_API_KEY=xxx export NETLAS_API_KEY=xxx export CRIMINALIP_API_KEY=xxx +export PUBLICWWW_API_KEY=xxx ``` -Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.org/login), [Netlas](https://app.netlas.io/registration/) and [CriminalIP](https://www.criminalip.io/register). +Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.org/login), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register) and [Publicwww](https://publicwww.com/profile/signup.html). ## Running Uncover diff --git a/runner/options.go b/runner/options.go index fe8c0ee6..13d12a14 100644 --- a/runner/options.go +++ b/runner/options.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/pkg/errors" "github.com/projectdiscovery/goflags" @@ -37,8 +36,7 @@ type Options struct { Verbose bool NoColor bool Timeout int - Delay int - delay time.Duration + RateLimit int Provider *Provider Retries int Shodan goflags.StringSlice @@ -82,7 +80,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.ProviderFile, "provider", "pc", defaultProviderConfigLocation, "provider configuration file"), flagSet.StringVar(&options.ConfigFile, "config", defaultConfigLocation, "flag configuration file"), flagSet.IntVar(&options.Timeout, "timeout", 30, "timeout in seconds"), - flagSet.IntVar(&options.Delay, "delay", 1, "delay between requests in seconds (0 to disable)"), + flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"), flagSet.IntVar(&options.Retries, "retry", 2, "number of times to retry a failed request"), ) @@ -266,12 +264,6 @@ func (options *Options) validateOptions() error { return errors.New("no engine specified") } - if options.Delay < 0 { - return errors.New("delay can't be negative") - } else { - options.delay = time.Duration(options.Delay) * time.Second - } - return nil } diff --git a/runner/runner.go b/runner/runner.go index a3422369..dcbc4565 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -162,7 +162,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return } - session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.Delay, r.options.Engine) + session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimit, r.options.Engine) if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) } diff --git a/uncover/session.go b/uncover/session.go index 55b098fa..4bb09fde 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -19,7 +19,7 @@ type Session struct { RateLimits *ratelimit.MultiLimiter } -func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Session, error) { +func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -49,8 +49,8 @@ func NewSession(keys *Keys, retryMax, timeout, delay int, engines []string) (*Se var err error rateLimitOpts := &ratelimit.Options{ MaxCount: 1, - Duration: time.Duration(delay), - IsUnlimited: delay == 0, + Duration: time.Duration(rateLimit), + IsUnlimited: rateLimit == 0, } rateLimitOpts.Key = engines[0] From 7e6fe500d5010ce2251025f954b002ba9d6cae1a Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 1 Mar 2023 21:21:57 +0100 Subject: [PATCH 40/58] removed session.RateLimits.Take(agent.Name()) --- uncover/agent/censys/censys.go | 4 ---- uncover/agent/criminalip/criminalip.go | 4 ---- uncover/agent/fofa/fofa.go | 4 ---- uncover/agent/hunter/hunter.go | 4 ---- uncover/agent/netlas/netlas.go | 5 ----- uncover/agent/publicwww/publicwww.go | 5 ----- uncover/agent/quake/quake.go | 5 ----- uncover/agent/shodan/shodan.go | 4 ---- uncover/agent/shodanidb/shodan.go | 4 ---- uncover/agent/zoomeye/zoomeye.go | 4 ---- 10 files changed, 43 deletions(-) diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 9acb988f..1d7125d4 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -65,10 +65,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest } request.Header.Set("Accept", "application/json") request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index e4384782..c5bdeb9c 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -63,10 +63,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, criminalipReq return nil, err } request.Header.Set("x-api-key", session.Keys.CriminalIPToken) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 1bbd60bd..4ece668f 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -67,10 +67,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest * return nil, err } request.Header.Set("Accept", "application/json") - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index 773ea328..cde321d6 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -93,9 +93,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, hunterRequest return nil, err } request.Header.Set("Accept", "application/json") - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index 9ca0cbf8..1475f162 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -92,10 +92,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon request.Header.Set("Content-Type", contentType) request.Header.Set("X-API-Key", session.Keys.NetlasToken) - - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index 88158fb8..fd2a71bc 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -116,10 +116,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon if err != nil { return nil, err } - - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 39f2c694..1dccab54 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -99,10 +99,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, quakeRequest request.Header.Set("Content-Type", "application/json") request.Header.Set("X-QuakeToken", session.Keys.QuakeToken) - - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index 6dd0ea6d..d20d93b1 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -64,10 +64,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index 16c0e2e8..b8eb947b 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -45,10 +45,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index 288d0815..9cfdb44c 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -67,10 +67,6 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, zoomeyeReques return nil, err } request.Header.Set("API-KEY", session.Keys.ZoomEyeToken) - err = session.RateLimits.Take(agent.Name()) - if err != nil { - return nil, err - } return session.Do(request) } From d06ea34a1a5dae58a12877412f79bd77fa046611 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Fri, 3 Mar 2023 14:18:37 +0100 Subject: [PATCH 41/58] added -rlm flag as nuclei --- runner/options.go | 56 +++++++++++++++++++++-------------------- runner/runner.go | 7 +++++- uncover/session.go | 6 ++--- uncover/session_test.go | 2 +- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/runner/options.go b/runner/options.go index 13d12a14..5930530e 100644 --- a/runner/options.go +++ b/runner/options.go @@ -22,33 +22,34 @@ var ( // Options contains the configuration options for tuning the enumeration process. type Options struct { - Query goflags.StringSlice - Engine goflags.StringSlice - ConfigFile string - ProviderFile string - OutputFile string - OutputFields string - JSON bool - Raw bool - Limit int - Silent bool - Version bool - Verbose bool - NoColor bool - Timeout int - RateLimit int - Provider *Provider - Retries int - Shodan goflags.StringSlice - ShodanIdb goflags.StringSlice - Fofa goflags.StringSlice - Censys goflags.StringSlice - Quake goflags.StringSlice - Netlas goflags.StringSlice - Hunter goflags.StringSlice - ZoomEye goflags.StringSlice - CriminalIP goflags.StringSlice - Publicwww goflags.StringSlice + Query goflags.StringSlice + Engine goflags.StringSlice + ConfigFile string + ProviderFile string + OutputFile string + OutputFields string + JSON bool + Raw bool + Limit int + Silent bool + Version bool + Verbose bool + NoColor bool + Timeout int + RateLimit int + RateLimitMinute int + Provider *Provider + Retries int + Shodan goflags.StringSlice + ShodanIdb goflags.StringSlice + Fofa goflags.StringSlice + Censys goflags.StringSlice + Quake goflags.StringSlice + Netlas goflags.StringSlice + Hunter goflags.StringSlice + ZoomEye goflags.StringSlice + CriminalIP goflags.StringSlice + Publicwww goflags.StringSlice } // ParseOptions parses the command line flags provided by a user @@ -81,6 +82,7 @@ func ParseOptions() *Options { flagSet.StringVar(&options.ConfigFile, "config", defaultConfigLocation, "flag configuration file"), flagSet.IntVar(&options.Timeout, "timeout", 30, "timeout in seconds"), flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"), + flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVar(&options.Retries, "retry", 2, "number of times to retry a failed request"), ) diff --git a/runner/runner.go b/runner/runner.go index dcbc4565..1197afe2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -162,7 +162,12 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return } - session, err := uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimit, r.options.Engine) + var session *uncover.Session + if r.options.RateLimitMinute > 0 { + session, err = uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimit, r.options.Engine, time.Minute) + } else if r.options.RateLimit > 0 { + session, err = uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimit, r.options.Engine, time.Second) + } if err != nil { gologger.Error().Label(agent.Name()).Msgf("couldn't create new session: %s\n", err) } diff --git a/uncover/session.go b/uncover/session.go index 4bb09fde..59e96187 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -19,7 +19,7 @@ type Session struct { RateLimits *ratelimit.MultiLimiter } -func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string) (*Session, error) { +func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, duration time.Duration) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -48,8 +48,8 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string) var err error rateLimitOpts := &ratelimit.Options{ - MaxCount: 1, - Duration: time.Duration(rateLimit), + MaxCount: uint(rateLimit), + Duration: duration, IsUnlimited: rateLimit == 0, } diff --git a/uncover/session_test.go b/uncover/session_test.go index 554d6692..c4c7aff8 100644 --- a/uncover/session_test.go +++ b/uncover/session_test.go @@ -19,7 +19,7 @@ func TestSessionRetry(t *testing.T) { })) ts := httptest.NewServer(router) engines := []string{"shoda,", "publicwww"} - session, err := NewSession(&Keys{}, 5, 3, 60, engines) + session, err := NewSession(&Keys{}, 5, 3, 60, engines, time.Second) require.Nil(t, err) req, err := retryablehttp.NewRequest(http.MethodGet, ts.URL, nil) require.Nil(t, err) From 2ef7b149d46da303a7aeb3c1511f2c5094d409b4 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Fri, 3 Mar 2023 16:17:08 +0100 Subject: [PATCH 42/58] session management --- runner/runner.go | 7 ++++--- uncover/session.go | 39 +++++++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 1197afe2..fc7b6730 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -17,6 +17,7 @@ import ( "github.com/projectdiscovery/uncover/uncover/agent/fofa" "github.com/projectdiscovery/uncover/uncover/agent/hunter" "github.com/projectdiscovery/uncover/uncover/agent/netlas" + "github.com/projectdiscovery/uncover/uncover/agent/publicwww" "github.com/projectdiscovery/uncover/uncover/agent/quake" "github.com/projectdiscovery/uncover/uncover/agent/shodan" "github.com/projectdiscovery/uncover/uncover/agent/shodanidb" @@ -115,7 +116,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { case "criminalip": agents = append(agents, &criminalip.Agent{}) case "publicwww": - agents = append(agents, &criminalip.Agent{}) + agents = append(agents, &publicwww.Agent{}) default: err = errors.New("unknown agent type") } @@ -164,8 +165,8 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { var session *uncover.Session if r.options.RateLimitMinute > 0 { - session, err = uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimit, r.options.Engine, time.Minute) - } else if r.options.RateLimit > 0 { + session, err = uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimitMinute, r.options.Engine, time.Minute) + } else { session, err = uncover.NewSession(&keys, r.options.Retries, r.options.Timeout, r.options.RateLimit, r.options.Engine, time.Second) } if err != nil { diff --git a/uncover/session.go b/uncover/session.go index 59e96187..ce5224e0 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -3,6 +3,7 @@ package uncover import ( "context" "crypto/tls" + "fmt" "net/http" "net/url" "time" @@ -39,6 +40,19 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, options.RetryWaitMax = time.Duration(timeout) * time.Second client := retryablehttp.NewWithHTTPClient(httpclient, options) + var defaultRateLimits = map[string]*ratelimit.Options{ + "shodan": {Key: "shodan", MaxCount: 1, Duration: time.Second, IsUnlimited: true}, + "shodan-idb": {Key: "shodan-idb", MaxCount: 1, Duration: time.Second, IsUnlimited: true}, + "fofa": {Key: "fofa", MaxCount: 1, Duration: time.Second, IsUnlimited: true}, + "censys": {Key: "censys", MaxCount: 1, Duration: 3 * time.Second, IsUnlimited: false}, + "quake": {Key: "quake", MaxCount: 1, Duration: time.Second, IsUnlimited: true}, + "hunter": {Key: "hunter", MaxCount: 15, Duration: time.Second, IsUnlimited: false}, + "zoomeye": {Key: "zoomeye", MaxCount: 1, Duration: time.Second, IsUnlimited: false}, + "netlas": {Key: "netlas", MaxCount: 1, Duration: time.Second, IsUnlimited: false}, + "criminalip": {Key: "criminalip", MaxCount: 1, Duration: time.Second, IsUnlimited: true}, + "publicwww": {Key: "publicwww", MaxCount: 1, Duration: time.Minute, IsUnlimited: false}, + } + session := &Session{ Client: client, Keys: keys, @@ -47,22 +61,23 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, } var err error - rateLimitOpts := &ratelimit.Options{ - MaxCount: uint(rateLimit), - Duration: duration, - IsUnlimited: rateLimit == 0, - } - - rateLimitOpts.Key = engines[0] - session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), rateLimitOpts) + session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), defaultRateLimits[engines[0]]) if err != nil { - return &Session{}, nil + return &Session{}, err } for _, engine := range engines[1:] { - engineOpts := rateLimitOpts - engineOpts.Key = engine - err := session.RateLimits.Add(engineOpts) + rateLimitOpts := defaultRateLimits[engine] + if rateLimitOpts == nil { + return nil, fmt.Errorf("no default rate limit found for engine %s", engine) + } + + if rateLimit > 0 { + rateLimitOpts.MaxCount = uint(rateLimit) + rateLimitOpts.Duration = duration + } + rateLimitOpts.IsUnlimited = rateLimit == 0 + err := session.RateLimits.Add(rateLimitOpts) if err != nil { return nil, err } From fd8bec3138909acf7a7e980ca6f83b9b4d903b0f Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Fri, 3 Mar 2023 16:26:42 +0100 Subject: [PATCH 43/58] fixes on session --- uncover/session.go | 33 +++++++++++++++++---------------- uncover/session_test.go | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/uncover/session.go b/uncover/session.go index ce5224e0..2e909252 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -60,26 +60,27 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, RateLimits: &ratelimit.MultiLimiter{}, } - var err error - session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), defaultRateLimits[engines[0]]) - if err != nil { - return &Session{}, err - } - - for _, engine := range engines[1:] { + for i, engine := range engines { + var err error rateLimitOpts := defaultRateLimits[engine] if rateLimitOpts == nil { return nil, fmt.Errorf("no default rate limit found for engine %s", engine) } - - if rateLimit > 0 { - rateLimitOpts.MaxCount = uint(rateLimit) - rateLimitOpts.Duration = duration - } - rateLimitOpts.IsUnlimited = rateLimit == 0 - err := session.RateLimits.Add(rateLimitOpts) - if err != nil { - return nil, err + if i == 0 { + session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), defaultRateLimits[engine]) + if err != nil { + return &Session{}, err + } + } else { + if rateLimit > 0 { + rateLimitOpts.MaxCount = uint(rateLimit) + rateLimitOpts.Duration = duration + } + rateLimitOpts.IsUnlimited = rateLimit == 0 + err = session.RateLimits.Add(rateLimitOpts) + if err != nil { + return nil, err + } } } diff --git a/uncover/session_test.go b/uncover/session_test.go index c4c7aff8..a0f7359f 100644 --- a/uncover/session_test.go +++ b/uncover/session_test.go @@ -18,7 +18,7 @@ func TestSessionRetry(t *testing.T) { t.Log("Slept for 10 seconds") })) ts := httptest.NewServer(router) - engines := []string{"shoda,", "publicwww"} + engines := []string{"shodan", "publicwww"} session, err := NewSession(&Keys{}, 5, 3, 60, engines, time.Second) require.Nil(t, err) req, err := retryablehttp.NewRequest(http.MethodGet, ts.URL, nil) From 43e11433d7d87a4a1ee4fdef864d1f89fdec9a7f Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Fri, 3 Mar 2023 23:44:38 +0100 Subject: [PATCH 44/58] hunter.how - new api provider support (#166) * started implementing hunterhow * wip: hunterhow integration * hunterhow integration * updated readme * used query.Limit * adding missing publicwww * missing publicwww in options * fixes, use genericutil for comparison --------- Co-authored-by: Mzack9999 --- README.md | 4 +- runner/options.go | 80 ++++++++++-------- runner/provider.go | 7 +- runner/runner.go | 11 ++- uncover/agent/hunterhow/hunterhow.go | 120 +++++++++++++++++++++++++++ uncover/agent/hunterhow/request.go | 31 +++++++ uncover/agent/hunterhow/response.go | 16 ++++ uncover/keys.go | 4 +- 8 files changed, 235 insertions(+), 38 deletions(-) create mode 100644 uncover/agent/hunterhow/hunterhow.go create mode 100644 uncover/agent/hunterhow/request.go create mode 100644 uncover/agent/hunterhow/response.go diff --git a/README.md b/README.md index 731aab7d..eae7cc75 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Usage: Flags: INPUT: -q, -query string[] search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt') - -e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan) + -e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan) SEARCH-ENGINE: -s, -shodan string[] search query for shodan (example: -shodan 'query.txt') @@ -83,6 +83,8 @@ SEARCH-ENGINE: -ze, -zoomeye string[] search query for zoomeye (example: -zoomeye 'query.txt') -ne, -netlas string[] search query for netlas (example: -netlas 'query.txt') -cl, -criminalip string[] search query for criminalip (example: -criminalip 'query.txt') + -pw, -publicwww string[] search query for publicwww (example: -publicwww 'query.txt') + -hh, -hunterhow string[] search query for hunterhow (example: -hunterhow 'query.txt') CONFIG: -pc, -provider string provider configuration file (default "$HOME/.config/uncover/provider-config.yaml") diff --git a/runner/options.go b/runner/options.go index fe8c0ee6..f6033f08 100644 --- a/runner/options.go +++ b/runner/options.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/gologger/levels" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" + genericutil "github.com/projectdiscovery/utils/generic" ) var ( @@ -51,6 +52,7 @@ type Options struct { ZoomEye goflags.StringSlice CriminalIP goflags.StringSlice Publicwww goflags.StringSlice + HunterHow goflags.StringSlice } // ParseOptions parses the command line flags provided by a user @@ -62,7 +64,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("input", "Input", flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions), - flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip) (default shodan)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow) (default shodan)", goflags.FileNormalizedStringSliceOptions), ) flagSet.CreateGroup("search-engine", "Search-Engine", @@ -76,6 +78,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.Netlas, "netlas", "ne", nil, "search query for netlas (example: -netlas 'query.txt')", goflags.FileStringSliceOptions), flagSet.StringSliceVarP(&options.CriminalIP, "criminalip", "cl", nil, "search query for criminalip (example: -criminalip 'query.txt')", goflags.FileStringSliceOptions), flagSet.StringSliceVarP(&options.Publicwww, "publicwww", "pw", nil, "search query for publicwww (example: -publicwww 'query.txt')", goflags.FileStringSliceOptions), + flagSet.StringSliceVarP(&options.HunterHow, "hunterhow", "hh", nil, "search query for hunterhow (example: -hunterhow 'query.txt')", goflags.FileStringSliceOptions), ) flagSet.CreateGroup("config", "Config", @@ -132,17 +135,19 @@ func ParseOptions() *Options { gologger.Warning().Msgf("couldn't parse env vars: %s\n", err) } - if len(options.Engine) == 0 && - len(options.Shodan) == 0 && - len(options.Censys) == 0 && - len(options.Quake) == 0 && - len(options.Fofa) == 0 && - len(options.ShodanIdb) == 0 && - len(options.Hunter) == 0 && - len(options.ZoomEye) == 0 && - len(options.Netlas) == 0 && - len(options.CriminalIP) == 0 && - len(options.Publicwww) == 0 { + if genericutil.EqualsAll(0, + len(options.Engine), + len(options.Shodan), + len(options.Censys), + len(options.Quake), + len(options.Fofa), + len(options.ShodanIdb), + len(options.Hunter), + len(options.ZoomEye), + len(options.Netlas), + len(options.CriminalIP), + len(options.Publicwww), + len(options.HunterHow)) { options.Engine = append(options.Engine, "shodan") } @@ -225,6 +230,9 @@ func (options *Options) loadProvidersFromEnv() error { if key, exists := os.LookupEnv("PUBLICWWW_API_KEY"); exists { options.Provider.Publicwww = append(options.Provider.Publicwww, key) } + if key, exists := os.LookupEnv("HUNTERHOW_API_KEY"); exists { + options.Provider.HunterHow = append(options.Provider.HunterHow, key) + } return nil } @@ -232,17 +240,19 @@ func (options *Options) loadProvidersFromEnv() error { func (options *Options) validateOptions() error { // Check if domain, list of domains, or stdin info was provided. // If none was provided, then return. - if len(options.Query) == 0 && - len(options.Shodan) == 0 && - len(options.Censys) == 0 && - len(options.Quake) == 0 && - len(options.Fofa) == 0 && - len(options.ShodanIdb) == 0 && - len(options.Hunter) == 0 && - len(options.ZoomEye) == 0 && - len(options.Netlas) == 0 && - len(options.CriminalIP) == 0 && - len(options.Publicwww) == 0 { + if genericutil.EqualsAll(0, + len(options.Query), + len(options.Shodan), + len(options.Censys), + len(options.Quake), + len(options.Fofa), + len(options.ShodanIdb), + len(options.Hunter), + len(options.ZoomEye), + len(options.Netlas), + len(options.CriminalIP), + len(options.Publicwww), + len(options.HunterHow)) { return errors.New("no query provided") } @@ -252,17 +262,19 @@ func (options *Options) validateOptions() error { } // Validate threads and options - if len(options.Engine) == 0 && - len(options.Shodan) == 0 && - len(options.Censys) == 0 && - len(options.Quake) == 0 && - len(options.Fofa) == 0 && - len(options.ShodanIdb) == 0 && - len(options.Hunter) == 0 && - len(options.ZoomEye) == 0 && - len(options.Netlas) == 0 && - len(options.CriminalIP) == 0 && - len(options.Publicwww) == 0 { + if genericutil.EqualsAll(0, + len(options.Engine), + len(options.Shodan), + len(options.Censys), + len(options.Quake), + len(options.Fofa), + len(options.ShodanIdb), + len(options.Hunter), + len(options.ZoomEye), + len(options.Netlas), + len(options.CriminalIP), + len(options.Publicwww), + len(options.HunterHow)) { return errors.New("no engine specified") } diff --git a/runner/provider.go b/runner/provider.go index 341df3fa..f95c6107 100644 --- a/runner/provider.go +++ b/runner/provider.go @@ -17,6 +17,7 @@ type Provider struct { Netlas []string `yaml:"netlas"` CriminalIP []string `yaml:"criminalip"` Publicwww []string `yaml:"publicwww"` + HunterHow []string `yaml:"hunterhow"` } func (provider *Provider) GetKeys() uncover.Keys { @@ -67,6 +68,9 @@ func (provider *Provider) GetKeys() uncover.Keys { if len(provider.Publicwww) > 0 { keys.PublicwwwToken = provider.Publicwww[rand.Intn(len(provider.Publicwww))] } + if len(provider.HunterHow) > 0 { + keys.HunterHowToken = provider.HunterHow[rand.Intn(len(provider.HunterHow))] + } return keys } @@ -79,5 +83,6 @@ func (provider *Provider) HasKeys() bool { len(provider.Hunter) > 0 || len(provider.ZoomEye) > 0 || len(provider.Netlas) > 0 || - len(provider.CriminalIP) > 0 + len(provider.CriminalIP) > 0 || + len(provider.HunterHow) > 0 } diff --git a/runner/runner.go b/runner/runner.go index dae918f2..db3aa013 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -17,6 +17,7 @@ import ( "github.com/projectdiscovery/uncover/uncover/agent/criminalip" "github.com/projectdiscovery/uncover/uncover/agent/fofa" "github.com/projectdiscovery/uncover/uncover/agent/hunter" + "github.com/projectdiscovery/uncover/uncover/agent/hunterhow" "github.com/projectdiscovery/uncover/uncover/agent/netlas" "github.com/projectdiscovery/uncover/uncover/agent/publicwww" "github.com/projectdiscovery/uncover/uncover/agent/quake" @@ -50,7 +51,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { return errors.New("no keys provided") } - var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter, publicwwwRatelimiter *ratelimit.Limiter + var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter, criminalipRatelimiter, publicwwwRatelimiter, hunterhowRateLimiter *ratelimit.Limiter if r.options.Delay > 0 { censysRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) fofaRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) @@ -62,6 +63,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { netlasRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) criminalipRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) publicwwwRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) + hunterhowRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay)) } else { censysRateLimiter = ratelimit.NewUnlimited(context.Background()) fofaRateLimiter = ratelimit.NewUnlimited(context.Background()) @@ -73,6 +75,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { netlasRatelimiter = ratelimit.NewUnlimited(context.Background()) criminalipRatelimiter = ratelimit.NewUnlimited(context.Background()) publicwwwRatelimiter = ratelimit.NewUnlimited(context.Background()) + hunterhowRateLimiter = ratelimit.NewUnlimited(context.Background()) } var agents []uncover.Agent @@ -116,6 +119,10 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { r.options.Engine = append(r.options.Engine, "publicwww") query = append(query, r.options.Publicwww...) } + if len(r.options.HunterHow) > 0 { + r.options.Engine = append(r.options.Engine, "hunterhow") + query = append(query, r.options.HunterHow...) + } // declare clients for _, engine := range r.options.Engine { @@ -144,6 +151,8 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { agent, err = criminalip.NewWithOptions(&uncover.AgentOptions{RateLimiter: criminalipRatelimiter}) case "publicwww": agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{RateLimiter: publicwwwRatelimiter}) + case "hunterhow": + agent, err = hunterhow.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterhowRateLimiter}) default: err = errors.New("unknown agent type") } diff --git a/uncover/agent/hunterhow/hunterhow.go b/uncover/agent/hunterhow/hunterhow.go new file mode 100644 index 00000000..ad7c77f4 --- /dev/null +++ b/uncover/agent/hunterhow/hunterhow.go @@ -0,0 +1,120 @@ +package hunterhow + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/projectdiscovery/uncover/uncover" +) + +const ( + baseURL = "https://api.hunter.how/" + baseEndpoint = "search" + Size = 100 +) + +type Agent struct { + options *uncover.AgentOptions +} + +func New() (uncover.Agent, error) { + return &Agent{}, nil +} + +func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { + return &Agent{options: options}, nil +} + +func (agent *Agent) Name() string { + return "hunterhow" +} + +func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan uncover.Result, error) { + if session.Keys.HunterHowToken == "" { + return nil, errors.New("empty hunterhow keys") + } + + results := make(chan uncover.Result) + + go func() { + defer close(results) + + numberOfResults := 0 + + pageQuery := 1 + + for { + hunterhowRequest := &Request{ + Query: query.Query, + PageSize: query.Limit, + Page: pageQuery, + } + + if numberOfResults > query.Limit { + break + } + + hunterhowResponse := agent.query(hunterhowRequest.buildURL(session.Keys.HunterHowToken), session, results) + if hunterhowResponse == nil { + break + } + + if len(hunterhowResponse) == 0 { + break + } + + numberOfResults += len(hunterhowResponse) + pageQuery += 1 + } + }() + + return results, nil +} + +func (agent *Agent) query(URL string, session *uncover.Session, results chan uncover.Result) []string { + resp, err := agent.queryURL(session, URL) + if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} + return nil + } + + var apiResponse Response + err = json.NewDecoder(resp.Body).Decode(&apiResponse) + if err != nil { + results <- uncover.Result{Source: agent.Name(), Error: err} + return nil + } + if apiResponse.Code != 200 { + results <- uncover.Result{Source: agent.Name(), Error: errors.New(apiResponse.Message)} + return nil + } + + var lines []string + for _, data := range apiResponse.Data.List { + result := uncover.Result{Source: agent.Name()} + result.Host = data.Domain + result.IP = data.IP + result.Port = data.Port + raw, _ := json.Marshal(result) + result.Raw = raw + results <- result + lines = append(lines, data.Domain) + } + + return lines +} + +func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Response, error) { + request, err := uncover.NewHTTPRequest( + http.MethodGet, + URL, + nil, + ) + if err != nil { + return nil, err + } + + agent.options.RateLimiter.Take() + return session.Do(request) +} diff --git a/uncover/agent/hunterhow/request.go b/uncover/agent/hunterhow/request.go new file mode 100644 index 00000000..0f27929f --- /dev/null +++ b/uncover/agent/hunterhow/request.go @@ -0,0 +1,31 @@ +package hunterhow + +import ( + "encoding/base64" + "strconv" + "time" +) + +type Request struct { + Query string `json:"query"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} + +func (r *Request) buildURL(key string) string { + timeFormat := "2006-01-02" + now := time.Now() + firstDay := time.Date(now.Year(), time.January, 1, 0, 0, 0, 0, now.Location()) + startTimeStr := firstDay.Format(timeFormat) + endTimeStr := now.Format(timeFormat) + + queryStr := baseURL + + baseEndpoint + "?api-key=" + key + + "&query=" + base64.StdEncoding.EncodeToString([]byte(r.Query)) + + "&start_time=" + startTimeStr + + "&end_time=" + endTimeStr + + "&page_size=" + strconv.Itoa(r.PageSize) + + "&page=" + strconv.Itoa(r.Page) + + return queryStr +} diff --git a/uncover/agent/hunterhow/response.go b/uncover/agent/hunterhow/response.go new file mode 100644 index 00000000..4ff2fae4 --- /dev/null +++ b/uncover/agent/hunterhow/response.go @@ -0,0 +1,16 @@ +package hunterhow + +type Response struct { + Code int `json:"code"` + Data Data `json:"data"` + Message string `json:"message"` +} +type List struct { + Domain string `json:"domain"` + IP string `json:"ip"` + Port int `json:"port"` +} +type Data struct { + List []List `json:"list"` + Total int `json:"total"` +} diff --git a/uncover/keys.go b/uncover/keys.go index 9a76a6bb..517f8610 100644 --- a/uncover/keys.go +++ b/uncover/keys.go @@ -12,6 +12,7 @@ type Keys struct { NetlasToken string CriminalIPToken string PublicwwwToken string + HunterHowToken string } func (keys Keys) Empty() bool { @@ -25,5 +26,6 @@ func (keys Keys) Empty() bool { keys.ZoomEyeToken == "" && keys.NetlasToken == "" && keys.CriminalIPToken == "" && - keys.PublicwwwToken == "" + keys.PublicwwwToken == "" && + keys.HunterHowToken == "" } From 69f0144d0ad3f410fb52ff0f28dc391f4601331d Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 6 Mar 2023 09:25:16 +0100 Subject: [PATCH 45/58] fixes --- runner/options.go | 61 +++++++++++++++------------- runner/runner.go | 4 +- uncover/agent/hunterhow/hunterhow.go | 14 +------ 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/runner/options.go b/runner/options.go index 37ca0095..fa2a850d 100644 --- a/runner/options.go +++ b/runner/options.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/pkg/errors" "github.com/projectdiscovery/goflags" @@ -23,35 +24,37 @@ var ( // Options contains the configuration options for tuning the enumeration process. type Options struct { - Query goflags.StringSlice - Engine goflags.StringSlice - ConfigFile string - ProviderFile string - OutputFile string - OutputFields string - JSON bool - Raw bool - Limit int - Silent bool - Version bool - Verbose bool - NoColor bool - Timeout int - Delay int - delay time.Duration - Provider *Provider - Retries int - Shodan goflags.StringSlice - ShodanIdb goflags.StringSlice - Fofa goflags.StringSlice - Censys goflags.StringSlice - Quake goflags.StringSlice - Netlas goflags.StringSlice - Hunter goflags.StringSlice - ZoomEye goflags.StringSlice - CriminalIP goflags.StringSlice - Publicwww goflags.StringSlice - HunterHow goflags.StringSlice + Query goflags.StringSlice + Engine goflags.StringSlice + ConfigFile string + ProviderFile string + OutputFile string + OutputFields string + JSON bool + Raw bool + Limit int + Silent bool + Version bool + Verbose bool + NoColor bool + Timeout int + Delay int + delay time.Duration + RateLimit int + RateLimitMinute int + Provider *Provider + Retries int + Shodan goflags.StringSlice + ShodanIdb goflags.StringSlice + Fofa goflags.StringSlice + Censys goflags.StringSlice + Quake goflags.StringSlice + Netlas goflags.StringSlice + Hunter goflags.StringSlice + ZoomEye goflags.StringSlice + CriminalIP goflags.StringSlice + Publicwww goflags.StringSlice + HunterHow goflags.StringSlice } // ParseOptions parses the command line flags provided by a user diff --git a/runner/runner.go b/runner/runner.go index 93d35a88..47bd94ee 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -121,9 +121,9 @@ func (r *Runner) Run(ctx context.Context, query ...string) error { case "criminalip": agents = append(agents, &criminalip.Agent{}) case "publicwww": - agent, err = publicwww.NewWithOptions(&uncover.AgentOptions{RateLimiter: publicwwwRatelimiter}) + agents = append(agents, &publicwww.Agent{}) case "hunterhow": - agent, err = hunterhow.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterhowRateLimiter}) + agents = append(agents, &hunterhow.Agent{}) default: err = errors.New("unknown agent type") } diff --git a/uncover/agent/hunterhow/hunterhow.go b/uncover/agent/hunterhow/hunterhow.go index ad7c77f4..fdea18cd 100644 --- a/uncover/agent/hunterhow/hunterhow.go +++ b/uncover/agent/hunterhow/hunterhow.go @@ -14,17 +14,7 @@ const ( Size = 100 ) -type Agent struct { - options *uncover.AgentOptions -} - -func New() (uncover.Agent, error) { - return &Agent{}, nil -} - -func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) { - return &Agent{options: options}, nil -} +type Agent struct{} func (agent *Agent) Name() string { return "hunterhow" @@ -114,7 +104,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon if err != nil { return nil, err } - - agent.options.RateLimiter.Take() return session.Do(request) } From 38bb5d54079949f8897cfbf783cb0bc2f71bce0e Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 6 Mar 2023 09:27:49 +0100 Subject: [PATCH 46/58] lint tests --- runner/options.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/runner/options.go b/runner/options.go index fa2a850d..a91f217b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/pkg/errors" "github.com/projectdiscovery/goflags" @@ -38,8 +37,6 @@ type Options struct { Verbose bool NoColor bool Timeout int - Delay int - delay time.Duration RateLimit int RateLimitMinute int Provider *Provider From fba304b5703765c163fbe84d6e204cedfd7ec80e Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 6 Mar 2023 10:16:59 +0100 Subject: [PATCH 47/58] adding missing rlm option --- README.md | 11 ++++++----- uncover/session.go | 11 ----------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b1dec908..d8eae720 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,12 @@ SEARCH-ENGINE: -hh, -hunterhow string[] search query for hunterhow (example: -hunterhow 'query.txt') CONFIG: - -pc, -provider string provider configuration file (default "$HOME/.config/uncover/provider-config.yaml") - -config string flag configuration file (default "$HOME/.config/uncover/config.yaml") - -timeout int timeout in seconds (default 30) - -rl, -rate-limit int maximum number of http requests to send per second - -retry int number of times to retry a failed request (default 2) + -pc, -provider string provider configuration file (default "$HOME/.config/uncover/provider-config.yaml") + -config string flag configuration file (default "$HOME/.config/uncover/config.yaml") + -timeout int timeout in seconds (default 30) + -rl, -rate-limit int maximum number of http requests to send per second + -rlm, -rate-limit-minute int maximum number of requests to send per minute + -retry int number of times to retry a failed request (default 2) OUTPUT: -o, -output string output file to write found results diff --git a/uncover/session.go b/uncover/session.go index 2e909252..2720c571 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -105,21 +105,10 @@ func (s *Session) Do(request *retryablehttp.Request, sources ...string) (*http.R requestURL, _ := url.QueryUnescape(request.URL.String()) return resp, errors.Errorf("unexpected status code %d received from %s", resp.StatusCode, requestURL) } - // var f *os.File - // var err error - // if _, _, ok := request.BasicAuth(); ok { - // f, err = os.Open("/Users/marcornvh/go/src/github.com/projectdiscovery/uncover/uncover/agent/censys/example.json") - // } else { - // f, err = os.Open("/Users/marcornvh/go/src/github.com/projectdiscovery/uncover/uncover/agent/shodan/example.json") - // } if err != nil { return nil, err } - // resp := &http.Response{ - // StatusCode: 200, - // Body: f, - // } return resp, nil } From 4fb24fe7f4a15cea76a55d510b0ad36a71a0a3bb Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 6 Mar 2023 10:27:51 +0100 Subject: [PATCH 48/58] small refactor --- uncover/session.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncover/session.go b/uncover/session.go index 2720c571..a6dfc892 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -60,14 +60,14 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, RateLimits: &ratelimit.MultiLimiter{}, } - for i, engine := range engines { + for _, engine := range engines { var err error rateLimitOpts := defaultRateLimits[engine] if rateLimitOpts == nil { return nil, fmt.Errorf("no default rate limit found for engine %s", engine) } - if i == 0 { - session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), defaultRateLimits[engine]) + if session.RateLimits == nil { + session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), rateLimitOpts) if err != nil { return &Session{}, err } @@ -75,8 +75,9 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, if rateLimit > 0 { rateLimitOpts.MaxCount = uint(rateLimit) rateLimitOpts.Duration = duration + } else { + rateLimitOpts.IsUnlimited = true } - rateLimitOpts.IsUnlimited = rateLimit == 0 err = session.RateLimits.Add(rateLimitOpts) if err != nil { return nil, err From 5c52df313a907ebfcc2ca21f98bcdb1b053ec839 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 6 Mar 2023 10:35:57 +0100 Subject: [PATCH 49/58] removing nil check --- uncover/session.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncover/session.go b/uncover/session.go index a6dfc892..21816d7e 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -60,13 +60,13 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, RateLimits: &ratelimit.MultiLimiter{}, } - for _, engine := range engines { + for i, engine := range engines { var err error rateLimitOpts := defaultRateLimits[engine] if rateLimitOpts == nil { return nil, fmt.Errorf("no default rate limit found for engine %s", engine) } - if session.RateLimits == nil { + if i == 0 { session.RateLimits, err = ratelimit.NewMultiLimiter(context.Background(), rateLimitOpts) if err != nil { return &Session{}, err From 2f7e2e303aa190d9a3c34e4438b7e4afc94247d6 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Mon, 6 Mar 2023 13:24:41 +0100 Subject: [PATCH 50/58] session fix --- uncover/agent/censys/censys.go | 2 +- uncover/agent/criminalip/criminalip.go | 2 +- uncover/agent/fofa/fofa.go | 2 +- uncover/agent/hunter/hunter.go | 2 +- uncover/agent/hunterhow/hunterhow.go | 2 +- uncover/agent/netlas/netlas.go | 2 +- uncover/agent/publicwww/publicwww.go | 2 +- uncover/agent/quake/quake.go | 2 +- uncover/agent/shodan/shodan.go | 2 +- uncover/agent/shodanidb/shodan.go | 2 +- uncover/agent/zoomeye/zoomeye.go | 2 +- uncover/session.go | 10 ++++------ uncover/session_test.go | 2 +- 13 files changed, 16 insertions(+), 18 deletions(-) diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index 1d7125d4..faf50d98 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -65,7 +65,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest } request.Header.Set("Accept", "application/json") request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - return session.Do(request) + return session.Do(request, agent.Name()) } func (agent *Agent) query(URL string, session *uncover.Session, censysRequest *CensysRequest, results chan uncover.Result) *CensysResponse { diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index c5bdeb9c..956c9b1f 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -63,7 +63,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, criminalipReq return nil, err } request.Header.Set("x-api-key", session.Keys.CriminalIPToken) - return session.Do(request) + return session.Do(request, agent.Name()) } func (agent *Agent) query(URL string, session *uncover.Session, criminalipRequest *CriminalIPRequest, results chan uncover.Result) *CriminalIPResponse { diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 4ece668f..3ad54930 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -67,7 +67,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest * return nil, err } request.Header.Set("Accept", "application/json") - return session.Do(request) + return session.Do(request, agent.Name()) } func (agent *Agent) query(URL string, session *uncover.Session, fofaRequest *FofaRequest, results chan uncover.Result) *FofaResponse { diff --git a/uncover/agent/hunter/hunter.go b/uncover/agent/hunter/hunter.go index cde321d6..34ec2b33 100644 --- a/uncover/agent/hunter/hunter.go +++ b/uncover/agent/hunter/hunter.go @@ -93,5 +93,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, hunterRequest return nil, err } request.Header.Set("Accept", "application/json") - return session.Do(request) + return session.Do(request, agent.Name()) } diff --git a/uncover/agent/hunterhow/hunterhow.go b/uncover/agent/hunterhow/hunterhow.go index fdea18cd..4761678e 100644 --- a/uncover/agent/hunterhow/hunterhow.go +++ b/uncover/agent/hunterhow/hunterhow.go @@ -104,5 +104,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon if err != nil { return nil, err } - return session.Do(request) + return session.Do(request, agent.Name()) } diff --git a/uncover/agent/netlas/netlas.go b/uncover/agent/netlas/netlas.go index 1475f162..64273b67 100644 --- a/uncover/agent/netlas/netlas.go +++ b/uncover/agent/netlas/netlas.go @@ -92,5 +92,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon request.Header.Set("Content-Type", contentType) request.Header.Set("X-API-Key", session.Keys.NetlasToken) - return session.Do(request) + return session.Do(request, agent.Name()) } diff --git a/uncover/agent/publicwww/publicwww.go b/uncover/agent/publicwww/publicwww.go index fd2a71bc..d5a5c210 100644 --- a/uncover/agent/publicwww/publicwww.go +++ b/uncover/agent/publicwww/publicwww.go @@ -116,5 +116,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string) (*http.Respon if err != nil { return nil, err } - return session.Do(request) + return session.Do(request, agent.Name()) } diff --git a/uncover/agent/quake/quake.go b/uncover/agent/quake/quake.go index 1dccab54..95f32dda 100644 --- a/uncover/agent/quake/quake.go +++ b/uncover/agent/quake/quake.go @@ -99,5 +99,5 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, quakeRequest request.Header.Set("Content-Type", "application/json") request.Header.Set("X-QuakeToken", session.Keys.QuakeToken) - return session.Do(request) + return session.Do(request, agent.Name()) } diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index d20d93b1..23c0b0d1 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -64,7 +64,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - return session.Do(request) + return session.Do(request, agent.Name()) } func (agent *Agent) query(URL string, session *uncover.Session, shodanRequest *ShodanRequest, results chan uncover.Result) *ShodanResponse { diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index b8eb947b..747f6a2d 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -45,7 +45,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest if err != nil { return nil, err } - return session.Do(request) + return session.Do(request, agent.Name()) } func (agent *Agent) query(URL string, session *uncover.Session, shodanRequest *ShodanRequest, results chan uncover.Result) { diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index 9cfdb44c..6c567df7 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -67,7 +67,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, zoomeyeReques return nil, err } request.Header.Set("API-KEY", session.Keys.ZoomEyeToken) - return session.Do(request) + return session.Do(request, agent.Name()) } func (agent *Agent) query(URL string, session *uncover.Session, zoomeyeRequest *ZoomEyeRequest, results chan uncover.Result) *ZoomEyeResponse { diff --git a/uncover/session.go b/uncover/session.go index 21816d7e..db4ea82d 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -88,12 +88,10 @@ func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, return session, nil } -func (s *Session) Do(request *retryablehttp.Request, sources ...string) (*http.Response, error) { - for _, source := range sources { - err := s.RateLimits.Take(source) - if err != nil { - return nil, err - } +func (s *Session) Do(request *retryablehttp.Request, source string) (*http.Response, error) { + err := s.RateLimits.Take(source) + if err != nil { + return nil, err } // close request connection (does not reuse connections) request.Close = true diff --git a/uncover/session_test.go b/uncover/session_test.go index a0f7359f..c373233a 100644 --- a/uncover/session_test.go +++ b/uncover/session_test.go @@ -23,7 +23,7 @@ func TestSessionRetry(t *testing.T) { require.Nil(t, err) req, err := retryablehttp.NewRequest(http.MethodGet, ts.URL, nil) require.Nil(t, err) - resp, err := session.Do(req) + resp, err := session.Do(req, engines[0]) t.Log(resp, err) require.ErrorContains(t, err, "giving up after 6 attempts") require.Nil(t, resp) From 559cf0128f25ca9f24366c39c5b518e2d4cd2770 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 7 Mar 2023 12:07:50 +0100 Subject: [PATCH 51/58] removing outdated errors package --- go.mod | 2 +- runner/options.go | 3 ++- runner/runner.go | 3 ++- uncover/agent/censys/censys.go | 2 +- uncover/agent/criminalip/criminalip.go | 2 +- uncover/agent/fofa/fofa.go | 2 +- uncover/agent/shodan/shodan.go | 2 +- uncover/agent/shodanidb/shodan.go | 3 ++- uncover/agent/zoomeye/zoomeye.go | 2 +- uncover/session.go | 3 +-- 10 files changed, 13 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index c4353c7a..e6d8f27c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/pkg/errors v0.9.1 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goflags v0.1.7 github.com/projectdiscovery/gologger v1.1.8 @@ -52,6 +51,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/runner/options.go b/runner/options.go index a91f217b..cc764a73 100644 --- a/runner/options.go +++ b/runner/options.go @@ -6,7 +6,8 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" + "errors" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" diff --git a/runner/runner.go b/runner/runner.go index 47bd94ee..d1b657f9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -9,7 +9,8 @@ import ( "sync" "time" - "github.com/pkg/errors" + "errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/uncover/uncover" "github.com/projectdiscovery/uncover/uncover/agent/censys" diff --git a/uncover/agent/censys/censys.go b/uncover/agent/censys/censys.go index faf50d98..029dd13f 100644 --- a/uncover/agent/censys/censys.go +++ b/uncover/agent/censys/censys.go @@ -6,7 +6,7 @@ import ( "net/http" "net/url" - "github.com/pkg/errors" + "errors" "github.com/projectdiscovery/uncover/uncover" ) diff --git a/uncover/agent/criminalip/criminalip.go b/uncover/agent/criminalip/criminalip.go index 956c9b1f..fb6a249b 100644 --- a/uncover/agent/criminalip/criminalip.go +++ b/uncover/agent/criminalip/criminalip.go @@ -6,7 +6,7 @@ import ( "net/http" "net/url" - "github.com/pkg/errors" + "errors" "github.com/projectdiscovery/uncover/uncover" ) diff --git a/uncover/agent/fofa/fofa.go b/uncover/agent/fofa/fofa.go index 3ad54930..45c519a9 100644 --- a/uncover/agent/fofa/fofa.go +++ b/uncover/agent/fofa/fofa.go @@ -7,7 +7,7 @@ import ( "net/http" "strconv" - "github.com/pkg/errors" + "errors" "github.com/projectdiscovery/uncover/uncover" ) diff --git a/uncover/agent/shodan/shodan.go b/uncover/agent/shodan/shodan.go index 23c0b0d1..e864a327 100644 --- a/uncover/agent/shodan/shodan.go +++ b/uncover/agent/shodan/shodan.go @@ -6,7 +6,7 @@ import ( "net/http" "net/url" - "github.com/pkg/errors" + "errors" "github.com/projectdiscovery/uncover/uncover" ) diff --git a/uncover/agent/shodanidb/shodan.go b/uncover/agent/shodanidb/shodan.go index 747f6a2d..c05faa52 100644 --- a/uncover/agent/shodanidb/shodan.go +++ b/uncover/agent/shodanidb/shodan.go @@ -6,7 +6,8 @@ import ( "net/http" "net/url" - "github.com/pkg/errors" + "errors" + "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/uncover/uncover" iputil "github.com/projectdiscovery/utils/ip" diff --git a/uncover/agent/zoomeye/zoomeye.go b/uncover/agent/zoomeye/zoomeye.go index 6c567df7..e9eef3dd 100644 --- a/uncover/agent/zoomeye/zoomeye.go +++ b/uncover/agent/zoomeye/zoomeye.go @@ -7,7 +7,7 @@ import ( "net/url" "strconv" - "github.com/pkg/errors" + "errors" "github.com/projectdiscovery/uncover/uncover" ) diff --git a/uncover/session.go b/uncover/session.go index db4ea82d..27cf1b97 100644 --- a/uncover/session.go +++ b/uncover/session.go @@ -8,7 +8,6 @@ import ( "net/url" "time" - "github.com/pkg/errors" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryablehttp-go" ) @@ -102,7 +101,7 @@ func (s *Session) Do(request *retryablehttp.Request, source string) (*http.Respo if resp.StatusCode != http.StatusOK { requestURL, _ := url.QueryUnescape(request.URL.String()) - return resp, errors.Errorf("unexpected status code %d received from %s", resp.StatusCode, requestURL) + return resp, fmt.Errorf("unexpected status code %d received from %s", resp.StatusCode, requestURL) } if err != nil { From 03facb89f6c306c1cbd1264c8abe237dc1663dbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 19:32:03 +0530 Subject: [PATCH 52/58] chore(deps): bump github.com/projectdiscovery/utils (#169) Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.13 to 0.0.14. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.13...v0.0.14) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c4353c7a..7a56d729 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect - github.com/projectdiscovery/utils v0.0.13 + github.com/projectdiscovery/utils v0.0.14 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index 26be39b6..d6e0a171 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQL github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.12 h1:kjeXJ4V6ZwgoMeKV2dINERjxAsBPB/p5+NV2aSQbrrU= github.com/projectdiscovery/retryablehttp-go v1.0.12/go.mod h1:dnJK347etdVKz7ljhemUhBK6EpDnVf0U1O8dGj7MM+0= -github.com/projectdiscovery/utils v0.0.13 h1:Bvguo7avXN+Zs9tHL+8+Qld65nwIE8Kr9qddErDDqmw= -github.com/projectdiscovery/utils v0.0.13/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo= +github.com/projectdiscovery/utils v0.0.14 h1:ZtyZu0stPQ/qQSw8hsB0pQll2mLOkuB4d08J3ZdbZPw= +github.com/projectdiscovery/utils v0.0.14/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= From c089b8170511ce891c8997228afc4a118b21b689 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 19:32:17 +0530 Subject: [PATCH 53/58] chore(deps): bump github.com/projectdiscovery/goflags (#170) Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.7 to 0.1.8. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.7...v0.1.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a56d729..dd1ec0a6 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/goflags v0.1.7 + github.com/projectdiscovery/goflags v0.1.8 github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 diff --git a/go.sum b/go.sum index d6e0a171..b2dac59a 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.7 h1:KvMs1KnXMWRhJiJj6K+cWurTWpbDYYlo+1cyGvf1YCc= -github.com/projectdiscovery/goflags v0.1.7/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= +github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g= +github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oTFX/xzGhVU= From caf82508b13c202d4c64001d630202e19369f4f2 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Thu, 16 Mar 2023 05:17:10 +0530 Subject: [PATCH 54/58] Add update check and self update flag (#173) (#174) --- go.mod | 49 +++++++++++---- go.sum | 152 ++++++++++++++++++++++++++++++++++------------ runner/banners.go | 12 +++- runner/options.go | 78 +++++++++++++++--------- 4 files changed, 209 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index 0ce1ddee..7a7587f4 100644 --- a/go.mod +++ b/go.mod @@ -16,43 +16,70 @@ require ( ) require ( - github.com/miekg/dns v1.1.50 // indirect - golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/tools v0.2.0 // indirect + github.com/miekg/dns v1.1.52 // indirect + golang.org/x/exp v0.0.0-20230314191032-db074128a8ec // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/tools v0.7.0 // indirect ) require ( - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/microcosm-cc/bluemonday v1.0.21 // indirect + github.com/microcosm-cc/bluemonday v1.0.23 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect - github.com/projectdiscovery/utils v0.0.14 - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( + aead.dev/minisign v0.2.0 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/alecthomas/chroma v0.10.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/glamour v0.6.0 // indirect + github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/dlclark/regexp2 v1.8.1 // indirect github.com/dsnet/compress v0.0.1 // indirect + github.com/fatih/color v1.14.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-github/v30 v30.1.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect + github.com/minio/selfupdate v0.6.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.1 // indirect github.com/nwaples/rardecode v1.1.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + github.com/yuin/goldmark v1.5.4 // indirect + github.com/yuin/goldmark-emoji v1.0.1 // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/oauth2 v0.5.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index b2dac59a..6d48a984 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,57 @@ +aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= +aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= +github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= +github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= +github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= +github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= +github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= +github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= @@ -39,19 +71,40 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= -github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= +github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= +github.com/miekg/dns v1.1.52 h1:Bmlc/qsNNULOe6bpXcUTsuOajd0DzRHwup6D9k1An0c= +github.com/miekg/dns v1.1.52/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= +github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= +github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= +github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -72,11 +125,15 @@ github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQL github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.12 h1:kjeXJ4V6ZwgoMeKV2dINERjxAsBPB/p5+NV2aSQbrrU= github.com/projectdiscovery/retryablehttp-go v1.0.12/go.mod h1:dnJK347etdVKz7ljhemUhBK6EpDnVf0U1O8dGj7MM+0= -github.com/projectdiscovery/utils v0.0.14 h1:ZtyZu0stPQ/qQSw8hsB0pQll2mLOkuB4d08J3ZdbZPw= -github.com/projectdiscovery/utils v0.0.14/go.mod h1:2CyxZXcx62NUiGJZZam23CpphqXy3kaomE9uvgHgkEo= +github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d h1:SH4/bKOzajIhyT5Urrm2VZMN5guHUCgvH6+vb1MJjhg= +github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d/go.mod h1:tmR53wBeGXNSuO7N2PQgVTVwMCxQFduPbelgBhkfyaI= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -92,52 +149,67 @@ github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= +github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= +github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws= -golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/exp v0.0.0-20230314191032-db074128a8ec h1:pAv+d8BM2JNnNctsLJ6nnZ6NqXT8N4+eauvZSb3P0I0= +golang.org/x/exp v0.0.0-20230314191032-db074128a8ec/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/runner/banners.go b/runner/banners.go index c2e223b6..df115909 100644 --- a/runner/banners.go +++ b/runner/banners.go @@ -2,6 +2,8 @@ package runner import ( "github.com/projectdiscovery/gologger" + updateutils "github.com/projectdiscovery/utils/update" + ) const banner = ` @@ -12,10 +14,18 @@ const banner = ` ` // Version is the current version of uncover -const Version = `v1.0.2` +const version = `v1.0.2` // showBanner is used to show the banner to the user func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") +} + +// GetUpdateCallback returns a callback function that updates uncover +func GetUpdateCallback() func() { + return func() { + showBanner() + updateutils.GetUpdateToolCallback("uncover", version)() + } } \ No newline at end of file diff --git a/runner/options.go b/runner/options.go index cc764a73..44e092b3 100644 --- a/runner/options.go +++ b/runner/options.go @@ -15,6 +15,7 @@ import ( fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" genericutil "github.com/projectdiscovery/utils/generic" + updateutils "github.com/projectdiscovery/utils/update" ) var ( @@ -24,35 +25,36 @@ var ( // Options contains the configuration options for tuning the enumeration process. type Options struct { - Query goflags.StringSlice - Engine goflags.StringSlice - ConfigFile string - ProviderFile string - OutputFile string - OutputFields string - JSON bool - Raw bool - Limit int - Silent bool - Version bool - Verbose bool - NoColor bool - Timeout int - RateLimit int - RateLimitMinute int - Provider *Provider - Retries int - Shodan goflags.StringSlice - ShodanIdb goflags.StringSlice - Fofa goflags.StringSlice - Censys goflags.StringSlice - Quake goflags.StringSlice - Netlas goflags.StringSlice - Hunter goflags.StringSlice - ZoomEye goflags.StringSlice - CriminalIP goflags.StringSlice - Publicwww goflags.StringSlice - HunterHow goflags.StringSlice + Query goflags.StringSlice + Engine goflags.StringSlice + ConfigFile string + ProviderFile string + OutputFile string + OutputFields string + JSON bool + Raw bool + Limit int + Silent bool + Version bool + Verbose bool + NoColor bool + Timeout int + RateLimit int + RateLimitMinute int + Provider *Provider + Retries int + Shodan goflags.StringSlice + ShodanIdb goflags.StringSlice + Fofa goflags.StringSlice + Censys goflags.StringSlice + Quake goflags.StringSlice + Netlas goflags.StringSlice + Hunter goflags.StringSlice + ZoomEye goflags.StringSlice + CriminalIP goflags.StringSlice + Publicwww goflags.StringSlice + HunterHow goflags.StringSlice + DisableUpdateCheck bool } // ParseOptions parses the command line flags provided by a user @@ -90,6 +92,11 @@ func ParseOptions() *Options { flagSet.IntVar(&options.Retries, "retry", 2, "number of times to retry a failed request"), ) + flagSet.CreateGroup("update", "Update", + flagSet.CallbackVarP(GetUpdateCallback(), "update", "up", "update uncover to latest version"), + flagSet.BoolVarP(&options.DisableUpdateCheck, "disable-update-check", "duc", false, "disable automatic uncover update check"), + ) + flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.OutputFile, "output", "o", "", "output file to write found results"), flagSet.StringVarP(&options.OutputFields, "field", "f", "ip:port", "field to display in output (ip,port,host)"), @@ -115,10 +122,21 @@ func ParseOptions() *Options { showBanner() if options.Version { - gologger.Info().Msgf("Current Version: %s\n", Version) + gologger.Info().Msgf("Current Version: %s\n", version) os.Exit(0) } + if !options.DisableUpdateCheck { + latestVersion, err := updateutils.GetVersionCheckCallback("uncover")() + if err != nil { + if options.Verbose { + gologger.Error().Msgf("uncover version check failed: %v", err.Error()) + } + } else { + gologger.Info().Msgf("Current uncover version %v %v", version, updateutils.GetVersionDescription(version, latestVersion)) + } + } + if options.ConfigFile != defaultConfigLocation { _ = options.loadConfigFrom(options.ConfigFile) } From f0e27ddfed925669defbe1b7a71196bdb077e2da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 05:23:57 +0530 Subject: [PATCH 55/58] chore(deps): bump github.com/projectdiscovery/retryablehttp-go (#171) Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.12 to 1.0.13. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.12...v1.0.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a7587f4..f44209a0 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/mapcidr v1.1.0 github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.12 + github.com/projectdiscovery/retryablehttp-go v1.0.13 github.com/stretchr/testify v1.8.2 ) diff --git a/go.sum b/go.sum index 6d48a984..c167da89 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/projectdiscovery/mapcidr v1.1.0 h1:Yeb+CGVsRYvHmZ9YSHb9iy4tzY9YuOm3oT github.com/projectdiscovery/mapcidr v1.1.0/go.mod h1:hck0bWXka5ZkUaBG+TWt99bzLy+4hAg9oANhEmm3GNs= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/retryablehttp-go v1.0.12 h1:kjeXJ4V6ZwgoMeKV2dINERjxAsBPB/p5+NV2aSQbrrU= -github.com/projectdiscovery/retryablehttp-go v1.0.12/go.mod h1:dnJK347etdVKz7ljhemUhBK6EpDnVf0U1O8dGj7MM+0= +github.com/projectdiscovery/retryablehttp-go v1.0.13 h1:gKxd/J08Dxc8a/LFvTz9+JUedEvivH3PoDnQQEHAY4M= +github.com/projectdiscovery/retryablehttp-go v1.0.13/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d h1:SH4/bKOzajIhyT5Urrm2VZMN5guHUCgvH6+vb1MJjhg= github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d/go.mod h1:tmR53wBeGXNSuO7N2PQgVTVwMCxQFduPbelgBhkfyaI= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From 19cf6b7cd548ab3e09549152e9b4ae6f0e64a2bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 05:29:03 +0530 Subject: [PATCH 56/58] chore(deps): bump github.com/projectdiscovery/utils (#175) Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.15-0.20230312150239-2528d5ff811d to 0.0.16. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/commits/v0.0.16) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f44209a0..156f13ee 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 // indirect - github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d + github.com/projectdiscovery/utils v0.0.16 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index c167da89..a04a272d 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQL github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryablehttp-go v1.0.13 h1:gKxd/J08Dxc8a/LFvTz9+JUedEvivH3PoDnQQEHAY4M= github.com/projectdiscovery/retryablehttp-go v1.0.13/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= -github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d h1:SH4/bKOzajIhyT5Urrm2VZMN5guHUCgvH6+vb1MJjhg= -github.com/projectdiscovery/utils v0.0.15-0.20230312150239-2528d5ff811d/go.mod h1:tmR53wBeGXNSuO7N2PQgVTVwMCxQFduPbelgBhkfyaI= +github.com/projectdiscovery/utils v0.0.16 h1:7vmi3haCyM3vk0yXSLjoid4p2/7bo042rcmG4Dtk+Sk= +github.com/projectdiscovery/utils v0.0.16/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= From 235a857367b1b574e340c89fbb2e80839b553870 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 16 Mar 2023 05:48:15 +0530 Subject: [PATCH 57/58] workflow update + version + readme update --- .github/workflows/build-test.yml | 5 ++++- .github/workflows/codeql-analysis.yml | 7 ++++--- .github/workflows/dockerhub-push.yml | 2 +- .github/workflows/lint-test.yml | 3 +++ .github/workflows/release-binary.yml | 6 ++++-- .goreleaser.yml | 6 +++++- Dockerfile | 18 +++++++++++++----- LICENSE => LICENSE.md | 0 README.md | 2 ++ runner/banners.go | 4 ++-- 10 files changed, 38 insertions(+), 15 deletions(-) rename LICENSE => LICENSE.md (100%) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 8b042502..b930cf41 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -2,6 +2,9 @@ name: 🔨 Build Test on: pull_request: + paths: + - '**.go' + - '**.mod' workflow_dispatch: jobs: @@ -30,7 +33,7 @@ jobs: working-directory: . - name: Race Condition Tests - run: go build -race . + run: go run -race . working-directory: cmd/uncover/ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9f533f8d..c0e7abd4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,10 +1,11 @@ name: 🚨 CodeQL Analysis on: - workflow_dispatch: pull_request: - branches: - - dev + paths: + - '**.go' + - '**.mod' + workflow_dispatch: jobs: analyze: diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 72f852d6..8efa97b0 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -9,7 +9,7 @@ on: jobs: docker: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-16-cores steps: - name: Git Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index d8e6e01b..b2fe9b7b 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -2,6 +2,9 @@ name: 🙏🏻 Lint Test on: pull_request: + paths: + - '**.go' + - '**.mod' workflow_dispatch: jobs: diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index df22c30f..a0483fb6 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -8,7 +8,7 @@ on: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" uses: actions/checkout@v3 @@ -28,4 +28,6 @@ jobs: workdir: . env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" \ No newline at end of file + SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" + DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 608fbef0..869348aa 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -39,4 +39,8 @@ announce: enabled: true channel: '#release' username: GoReleaser - message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' \ No newline at end of file + message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' + + discord: + enabled: true + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e89a6e5d..edcad476 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,15 @@ -FROM golang:1.18.2-alpine3.14 AS build-env -RUN go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest +# Base +FROM golang:1.20.2-alpine AS builder +RUN apk add --no-cache git build-base +WORKDIR /app +COPY . /app +RUN go mod download +RUN go build -o ./cmd/uncover ./cmd/uncover +# Release FROM alpine:3.17.2 -RUN apk add --no-cache bind-tools ca-certificates -COPY --from=build-env /go/bin/uncover /usr/local/bin/uncover -ENTRYPOINT ["uncover"] +RUN apk -U upgrade --no-cache \ + && apk add --no-cache bind-tools ca-certificates +COPY --from=builder /app/cmd/uncover/uncover /usr/local/bin/ + +ENTRYPOINT ["uncover"] \ No newline at end of file diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md diff --git a/README.md b/README.md index d8eae720..21b30eca 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ - **[Zoomeye](https://www.zoomeye.org)** - **[Netlas](https://netlas.io/)** - **[CriminalIP](https://www.criminalip.io)** + - **[PublicWWW](https://publicwww.com)** + - **[HunterHow](https://hunter.how)** - Multiple API key input support - Automatic API key randomization - **stdin** / **stdout** support for input diff --git a/runner/banners.go b/runner/banners.go index df115909..e461b87a 100644 --- a/runner/banners.go +++ b/runner/banners.go @@ -10,11 +10,11 @@ const banner = ` __ ______ _________ _ _____ _____ / / / / __ \/ ___/ __ \ | / / _ \/ ___/ / /_/ / / / / /__/ /_/ / |/ / __/ / -\__,_/_/ /_/\___/\____/|___/\___/_/ v1.0.2 +\__,_/_/ /_/\___/\____/|___/\___/_/ ` // Version is the current version of uncover -const version = `v1.0.2` +const version = `v1.0.3` // showBanner is used to show the banner to the user func showBanner() { From eccd0f910df2072fd5660607a30522b0d7517e16 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 16 Mar 2023 05:52:39 +0530 Subject: [PATCH 58/58] workflow update --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b930cf41..58cba591 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -33,7 +33,7 @@ jobs: working-directory: . - name: Race Condition Tests - run: go run -race . + run: go build -race . working-directory: cmd/uncover/