From c73978683593d0fc861a6d67ac8caef7f4b96782 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 16 Sep 2021 17:35:47 +0200 Subject: [PATCH 01/67] Adding socks5 proxy support --- v2/pkg/runner/options.go | 2 ++ v2/pkg/runner/runner.go | 3 ++- v2/pkg/scan/option.go | 1 + v2/pkg/scan/scan.go | 27 +++++++++++++++++++++++++-- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index dc97c240..c320720a 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -46,6 +46,7 @@ type Options struct { ScanAllIPS bool // Scan all the ips ScanType string // Scan Type config *ConfigFile + Proxy string // Socks5 proxy } // ParseOptions parses the command line flags provided by a user @@ -84,6 +85,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan") flag.BoolVar(&options.ScanAllIPS, "scan-all-ips", false, "Scan all the ips") flag.StringVar(&options.ScanType, "s", SynScan, "Scan Type (s - SYN, c - CONNECT)") + flag.StringVar(&options.Proxy, "proxy", "", "Socks5 proxy") flag.Parse() diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index f2df4be2..e0c8bdb4 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -57,6 +57,7 @@ func NewRunner(options *Options) (*Runner, error) { Root: isRoot(), ExcludeCdn: options.ExcludeCDN, ExcludedIps: excludedIps, + Proxy: options.Proxy, }) if err != nil { return nil, err @@ -278,7 +279,7 @@ func (r *Runner) handleHostPort(host string, port int) { return } - open, err := scan.ConnectPort(host, port, time.Duration(r.options.Timeout)*time.Millisecond) + open, err := r.scanner.ConnectPort(host, port, time.Duration(r.options.Timeout)*time.Millisecond) if open && err == nil { r.scanner.ScanResults.AddPort(host, port) } diff --git a/v2/pkg/scan/option.go b/v2/pkg/scan/option.go index 4a0fff09..cc1ddc0c 100644 --- a/v2/pkg/scan/option.go +++ b/v2/pkg/scan/option.go @@ -13,4 +13,5 @@ type Options struct { Root bool ExcludeCdn bool ExcludedIps []string + Proxy string } diff --git a/v2/pkg/scan/scan.go b/v2/pkg/scan/scan.go index 3f2186d8..3cac29af 100644 --- a/v2/pkg/scan/scan.go +++ b/v2/pkg/scan/scan.go @@ -2,6 +2,7 @@ package scan import ( "fmt" + "log" "math/rand" "net" "strings" @@ -17,6 +18,7 @@ import ( "github.com/projectdiscovery/networkpolicy" "golang.org/x/net/icmp" "golang.org/x/net/ipv4" + "golang.org/x/net/proxy" ) // State determines the internal scan state @@ -56,6 +58,7 @@ type Scanner struct { rate int listenPort int timeout time.Duration + proxyDialer proxy.Dialer Ports []int IPRanger *ipranger.IPRanger @@ -143,6 +146,14 @@ func NewScanner(options *Options) (*Scanner, error) { } } + if options.Proxy != "" { + proxyDialer, err := proxy.SOCKS5("tcp", options.Proxy, nil, &net.Dialer{Timeout: options.Timeout}) + if err != nil { + log.Fatal(err) + } + scanner.proxyDialer = proxyDialer + } + return scanner, nil } @@ -355,8 +366,20 @@ func GetInterfaceFromIP(ip net.IP) (*net.Interface, error) { } // ConnectPort a single host and port -func ConnectPort(host string, port int, timeout time.Duration) (bool, error) { - conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), timeout) +func (s *Scanner) ConnectPort(host string, port int, timeout time.Duration) (bool, error) { + hostport := net.JoinHostPort(host, fmt.Sprint(port)) + var ( + err error + conn net.Conn + ) + if s.proxyDialer != nil { + conn, err = s.proxyDialer.Dial("tcp", hostport) + if err != nil { + log.Fatal(err) + } + } else { + conn, err = net.DialTimeout("tcp", hostport, timeout) + } if err != nil { return false, err } From 0dab885f096a09d9d649592f4b6fd1c199790784 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 18 Oct 2021 14:00:45 +0200 Subject: [PATCH 02/67] fixing merge error --- v2/pkg/runner/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 2ae79c48..7a27743e 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -95,7 +95,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Nmap, "nmap", false, "Invoke nmap scan on targets (nmap must be installed) - Deprecated"), flagSet.StringVar(&options.NmapCLI, "nmap-cli", "", "nmap command to run on found results (example: -nmap-cli 'nmap -sV')"), flagSet.StringVar(&options.Resolvers, "r", "", "Custom resolvers to use to resolve DNS names (comma separated or from file)"), - flagSet.StringVar(&options.Proxy, "proxy", "", "Socks5 proxy") + flagSet.StringVar(&options.Proxy, "proxy", "", "Socks5 proxy"), ) createGroup(flagSet, "optimization", "Optimization", @@ -116,7 +116,7 @@ func ParseOptions() *Options { ) _ = flagSet.Parse() - + // Check if stdin pipe was given options.Stdin = hasStdin() From 82f810de1c4bcd8fb64f24ae19d01aec8168c019 Mon Sep 17 00:00:00 2001 From: Passer6y <34622501+Passer6y@users.noreply.github.com> Date: Mon, 25 Oct 2021 15:39:33 +0800 Subject: [PATCH 03/67] Supports comma separated host scanning --- v2/pkg/runner/targets.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/runner/targets.go b/v2/pkg/runner/targets.go index 9238f1f8..3e6d7521 100644 --- a/v2/pkg/runner/targets.go +++ b/v2/pkg/runner/targets.go @@ -44,7 +44,14 @@ func (r *Runner) mergeToFile() (string, error) { // target defined via CLI argument if r.options.Host != "" { - fmt.Fprintf(tempInput, "%s\n", r.options.Host) + if strings.Contains(r.Options.Host, ","){ + splitHosts := strings.Split(r.Options.Host, ",") + for _, host := range splitHosts{ + fmt.Fprintf(tempInput, "%s\n", host) + } + }else{ + fmt.Fprintf(tempInput, "%s\n", r.Options.Host) + } } // Targets from file From 7c8177985d16324fc04410392e353371f8da5cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 13:05:46 +0000 Subject: [PATCH 04/67] chore(deps): bump golang from 1.17.2-alpine to 1.17.3-alpine Bumps golang from 1.17.2-alpine to 1.17.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a396b71c..526f48f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.2-alpine AS builder +FROM golang:1.17.3-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From 02b26c457812008ec462b2ce5d994fa6ab04bd4a Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 17 Nov 2021 15:49:55 -0600 Subject: [PATCH 05/67] functional test implementation --- v2/cmd/functional-test/main.go | 83 ++++++++++++++++++++++++++++ v2/cmd/functional-test/run.sh | 13 +++++ v2/cmd/functional-test/testcases.txt | 19 +++++++ 3 files changed, 115 insertions(+) create mode 100644 v2/cmd/functional-test/main.go create mode 100755 v2/cmd/functional-test/run.sh create mode 100644 v2/cmd/functional-test/testcases.txt diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go new file mode 100644 index 00000000..09fb35ab --- /dev/null +++ b/v2/cmd/functional-test/main.go @@ -0,0 +1,83 @@ +package main + + import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + + "github.com/projectdiscovery/naabu/v2/cmd/internal/testutils" + ) + + var ( + debug = os.Getenv("DEBUG") == "true" + success = aurora.Green("[✓]").String() + failed = aurora.Red("[✘]").String() + errored = false + + mainNaabuBinary = flag.String("main", "", "Main Branch Naabu Binary") + devNaabuBinary = flag.String("dev", "", "Dev Branch Naabu Binary") + testcases = flag.String("testcases", "", "Test cases file for Naabu functional tests") + ) + + func main() { + flag.Parse() + + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } + } + + func runFunctionalTests() error { + file, err := os.Open(*testcases) + if err != nil { + return errors.Wrap(err, "could not open test cases") + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + if err := runIndividualTestCase(text); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, text) + } + } + return nil + } + + func runIndividualTestCase(testcase string) error { + parts := strings.Fields(testcase) + + var finalArgs []string + var target string + if len(parts) > 1 { + finalArgs = parts[2:] + target = parts[0] + } + mainOutput, err := testutils.RunNaabuBinaryAndGetResults(target, *mainNaabuBinary, debug, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run naabu main test") + } + devOutput, err := testutils.RunNaabuBinaryAndGetResults(target, *devNaabuBinary, debug, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run naabu dev test") + } + if len(mainOutput) == len(devOutput) { + return nil + } + return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) + } diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh new file mode 100755 index 00000000..9527cb30 --- /dev/null +++ b/v2/cmd/functional-test/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + + echo 'Building functional-test binary' + go build + + echo 'Building NAABU binary from current branch' + go build -o naabu_dev ../naabu + + echo 'Installing latest release of NAABU' + GO111MODULE=on go build -v github.com/projectdiscovery/naabu/v2/cmd/naabu + + echo 'Starting NAABU functional test' + ./functional-test -main ./naabu -dev ./naabu_dev -testcases testcases.txt diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt new file mode 100644 index 00000000..3cb64c87 --- /dev/null +++ b/v2/cmd/functional-test/testcases.txt @@ -0,0 +1,19 @@ +example.com {{binary}} -silent +example.com {{binary}} -silent -il +example.com {{binary}} -silent l test-data/request.txt +example.com {{binary}} -silent -p 80,443 +example.com {{binary}} -silent -tp 100 +example.com {{binary}} -silent -ep 200 +example.com {{binary}} -silent -ec 100,400 +example.com {{binary}} -silent -version +example.com {{binary}} -silent -retries 5 +example.com {{binary}} -silent -source-ip +example.com {{binary}} -silent -c 25 +example.com {{binary}} -silent -verify +example.com {{binary}} -silent -r test-data/request.txt +example.com {{binary}} -silent -warm-up-time 2 +example.com {{binary}} -silent -nmap-cli 'nmap -sV' +example.com {{binary}} -silent -c 25 + + + From e9dbeb4f1961d20c206e2f1fdb41e506882917fa Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 17 Nov 2021 15:50:41 -0600 Subject: [PATCH 06/67] functional tests --- .github/workflows/functional-test.yml | 25 +++++++++ .gitignore | 6 ++- v2/cmd/internal/testutils/integration.go | 67 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/functional-test.yml create mode 100644 v2/cmd/internal/testutils/integration.go diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 00000000..59c0013d --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,25 @@ +name: 🧪 Functional Test + on: + push: + pull_request: + workflow_dispatch: + + + jobs: + functional: + name: Functional Test + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh + working-directory: v2/cmd/functional-test diff --git a/.gitignore b/.gitignore index 74400b11..9f26f6ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ cmd/naabu/naabu* v2/cmd/naabu/naabu* -vendor \ No newline at end of file +vendor +v2/cmd/functional-test/naabu_dev +v2/cmd/functional-test/functional-test +v2/cmd/functional-test/naabu +v2/cmd/functional-test/*.cfg diff --git a/v2/cmd/internal/testutils/integration.go b/v2/cmd/internal/testutils/integration.go new file mode 100644 index 00000000..510c9e1a --- /dev/null +++ b/v2/cmd/internal/testutils/integration.go @@ -0,0 +1,67 @@ +package testutils + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +// RunNaabuAndGetResults returns a list of results +func RunNaabuAndGetResults(question string, debug bool, extra ...string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := `echo "` + question + `" | ./naabu ` + cmdLine += strings.Join(extra, " ") + if debug { + cmdLine += " -debug" + cmd.Stderr = os.Stderr + } else { + cmdLine += " -silent" + } + + cmd.Args = append(cmd.Args, cmdLine) + + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} +func RunNaabuBinaryAndGetResults(target string, naabuBinary string, debug bool, args []string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf(`echo %s | %s `, target, naabuBinary) + cmdLine += strings.Join(args, " ") + if debug { + cmdLine += " -debug" + cmd.Stderr = os.Stderr + } else { + cmdLine += " -silent" + } + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts,nil +} + +// TestCase is a single integration test case +type TestCase interface { + // Execute executes a test case and returns any errors if occurred + Execute() error +} From 628a4aa4af065bf19d070bdc1beb2be3189a4f03 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 23 Nov 2021 14:56:07 -0600 Subject: [PATCH 07/67] testcases update --- v2/cmd/functional-test/test-data/request.txt | 1 + v2/cmd/functional-test/testcases.txt | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 v2/cmd/functional-test/test-data/request.txt diff --git a/v2/cmd/functional-test/test-data/request.txt b/v2/cmd/functional-test/test-data/request.txt new file mode 100644 index 00000000..de54ac66 --- /dev/null +++ b/v2/cmd/functional-test/test-data/request.txt @@ -0,0 +1 @@ +example.com diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 3cb64c87..c773dd21 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -7,13 +7,17 @@ example.com {{binary}} -silent -ep 200 example.com {{binary}} -silent -ec 100,400 example.com {{binary}} -silent -version example.com {{binary}} -silent -retries 5 -example.com {{binary}} -silent -source-ip example.com {{binary}} -silent -c 25 example.com {{binary}} -silent -verify +example.com {{binary}} -silent -timeout 30 example.com {{binary}} -silent -r test-data/request.txt example.com {{binary}} -silent -warm-up-time 2 example.com {{binary}} -silent -nmap-cli 'nmap -sV' example.com {{binary}} -silent -c 25 +www.projectdiscovery.io {{binary}} -silent -eh 127.0.0.1:8082,127.0.0.1:6089 +www.projectdiscovery.io {{binary}} -silent -stats +1.1.1.1 {{binary}} -silent -s CONNECT +1.1.1.1 {{binary}} -silent -json From e2f146102ec065882265e6511b465e48497ff352 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 24 Nov 2021 11:10:02 -0600 Subject: [PATCH 08/67] workflow update for functional test --- .github/workflows/functional-test.yml | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 59c0013d..d4e21bf4 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -1,25 +1,25 @@ name: 🧪 Functional Test - on: - push: - pull_request: - workflow_dispatch: +on: + push: + pull_request: + workflow_dispatch: - jobs: - functional: - name: Functional Test - runs-on: ubuntu-latest - steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 +jobs: + functional: + name: Functional Test + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 - - name: Check out code - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v2 - - name: Functional Tests - run: | - chmod +x run.sh - bash run.sh - working-directory: v2/cmd/functional-test + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh + working-directory: v2/cmd/functional-test From a80b02990f393bdf0d2b4dc2e1e12d6bb0ec124a Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 24 Nov 2021 13:25:36 -0600 Subject: [PATCH 09/67] testcase changes --- v2/cmd/functional-test/testcases.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index c773dd21..bc022b5c 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -16,8 +16,9 @@ example.com {{binary}} -silent -nmap-cli 'nmap -sV' example.com {{binary}} -silent -c 25 www.projectdiscovery.io {{binary}} -silent -eh 127.0.0.1:8082,127.0.0.1:6089 www.projectdiscovery.io {{binary}} -silent -stats -1.1.1.1 {{binary}} -silent -s CONNECT 1.1.1.1 {{binary}} -silent -json +1.1.1.1 {{binary}} -silent -s CONNECT + From 47ba9c737f9dd8e329f9957d51ea73a084c55b76 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 24 Nov 2021 13:36:11 -0600 Subject: [PATCH 10/67] added install libpcap in functional test workflow --- .github/workflows/functional-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index d4e21bf4..427904ac 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: -jobs: +jobs: functional: name: Functional Test runs-on: ubuntu-latest @@ -15,6 +15,9 @@ jobs: with: go-version: 1.17 + - name: Install libpcap-dev + run: sudo apt install libpcap-dev + - name: Check out code uses: actions/checkout@v2 From d0d37be251dde2043745db09ebe5d5eec7705782 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 29 Nov 2021 00:08:32 +0100 Subject: [PATCH 11/67] Adding tests for runner module --- .github/workflows/build-test.yml | 15 +++-- v2/go.mod | 10 ++- v2/pkg/result/results.go | 6 +- v2/pkg/result/results_test.go | 64 ++++++++++++++++++ v2/pkg/runner/banners_test.go | 12 ++++ v2/pkg/runner/ips_test.go | 39 +++++++++++ v2/pkg/runner/nmap.go | 16 +++-- v2/pkg/runner/nmap_test.go | 25 +++++++ v2/pkg/runner/output_test.go | 28 ++++++++ v2/pkg/runner/ports.go | 22 +++--- v2/pkg/runner/ports_test.go | 111 +++++++++++++++++++++++++++++++ v2/pkg/runner/runner.go | 4 +- v2/pkg/runner/util.go | 7 +- v2/pkg/runner/util_test.go | 42 ++++++++++++ v2/pkg/runner/validate.go | 17 +++-- v2/pkg/runner/validate_test.go | 22 ++++++ 16 files changed, 403 insertions(+), 37 deletions(-) create mode 100644 v2/pkg/result/results_test.go create mode 100644 v2/pkg/runner/banners_test.go create mode 100644 v2/pkg/runner/ips_test.go create mode 100644 v2/pkg/runner/nmap_test.go create mode 100644 v2/pkg/runner/output_test.go create mode 100644 v2/pkg/runner/ports_test.go create mode 100644 v2/pkg/runner/util_test.go create mode 100644 v2/pkg/runner/validate_test.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 91d17df6..ed205d66 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -15,15 +15,22 @@ jobs: with: go-version: 1.17 + - name: Check out code + uses: actions/checkout@v2 + - name: Install libpcap-dev run: sudo apt install libpcap-dev - - name: Check out code - uses: actions/checkout@v2 + - name: Install nmap + run: sudo apt install nmap - name: Test - run: go test . - working-directory: v2/cmd/naabu/ + run: go test ./... + working-directory: v2/ + + # - name: Integration Tests + # run: go run . + # working-directory: v2/cmd/integration_tests/ - name: Build run: go build . diff --git a/v2/go.mod b/v2/go.mod index 55f14fe0..3b942e36 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -22,19 +22,25 @@ require ( go.uber.org/ratelimit v0.2.0 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) + +require ( + github.com/pkg/errors v0.9.1 + github.com/stretchr/testify v1.7.0 ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.43 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/hmap v0.0.1 // indirect github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect diff --git a/v2/pkg/result/results.go b/v2/pkg/result/results.go index ad42acdb..a59ae309 100644 --- a/v2/pkg/result/results.go +++ b/v2/pkg/result/results.go @@ -12,8 +12,8 @@ type Result struct { // NewResult structure func NewResult() *Result { ipPorts := make(map[string]map[int]struct{}) - ipDomains := make(map[string]struct{}) - return &Result{IPPorts: ipPorts, IPS: ipDomains} + ips := make(map[string]struct{}) + return &Result{IPPorts: ipPorts, IPS: ips} } // AddPort to a specific ip @@ -26,6 +26,7 @@ func (r *Result) AddPort(k string, v int) { } r.IPPorts[k][v] = struct{}{} + r.IPS[k] = struct{}{} } // SetPorts for a specific ip @@ -34,6 +35,7 @@ func (r *Result) SetPorts(k string, v map[int]struct{}) { defer r.Unlock() r.IPPorts[k] = v + r.IPS[k] = struct{}{} } // IPHasPort checks if an ip has a specific port diff --git a/v2/pkg/result/results_test.go b/v2/pkg/result/results_test.go new file mode 100644 index 00000000..e94dca01 --- /dev/null +++ b/v2/pkg/result/results_test.go @@ -0,0 +1,64 @@ +package result + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAddPort(t *testing.T) { + targetIP := "127.0.0.1" + targetPort := 8080 + targetPorts := map[int]struct{}{targetPort: {}} + + res := NewResult() + res.AddPort(targetIP, targetPort) + + expectedIPS := map[string]struct{}{targetIP: {}} + assert.Equal(t, res.IPS, expectedIPS) + + expectedIPSPorts := map[string]map[int]struct{}{targetIP: targetPorts} + assert.Equal(t, res.IPPorts, expectedIPSPorts) +} + +func TestSetPorts(t *testing.T) { + targetIP := "127.0.0.1" + targetPorts := map[int]struct{}{80: {}, 8080: {}} + + res := NewResult() + res.SetPorts(targetIP, targetPorts) + + expectedIPS := map[string]struct{}{targetIP: {}} + assert.Equal(t, res.IPS, expectedIPS) + + expectedIPSPorts := map[string]map[int]struct{}{targetIP: targetPorts} + assert.Equal(t, res.IPPorts, expectedIPSPorts) +} + +func TestIPHasPort(t *testing.T) { + targetIP := "127.0.0.1" + targetPort := 8080 + + res := NewResult() + res.AddPort(targetIP, targetPort) + assert.True(t, res.IPHasPort(targetIP, targetPort)) + assert.False(t, res.IPHasPort(targetIP, 1111)) +} + +func TestSetIP(t *testing.T) { + targetIP := "127.0.0.1" + + res := NewResult() + res.SetIP(targetIP) + expectedIPS := map[string]struct{}{targetIP: {}} + assert.Equal(t, res.IPS, expectedIPS) +} + +func TestHasIP(t *testing.T) { + targetIP := "127.0.0.1" + + res := NewResult() + res.SetIP(targetIP) + assert.True(t, res.HasIP(targetIP)) + assert.False(t, res.HasIP("1.2.3.4")) +} diff --git a/v2/pkg/runner/banners_test.go b/v2/pkg/runner/banners_test.go new file mode 100644 index 00000000..adfb24ff --- /dev/null +++ b/v2/pkg/runner/banners_test.go @@ -0,0 +1,12 @@ +package runner + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShowNetworkInterfaces(t *testing.T) { + // non root users should be able to list interfaces + assert.Nil(t, showNetworkInterfaces()) +} diff --git a/v2/pkg/runner/ips_test.go b/v2/pkg/runner/ips_test.go new file mode 100644 index 00000000..360e8cf5 --- /dev/null +++ b/v2/pkg/runner/ips_test.go @@ -0,0 +1,39 @@ +package runner + +import ( + "os" + "strings" + "testing" + + "github.com/projectdiscovery/fileutil" + "github.com/stretchr/testify/assert" +) + +func TestParseExcludedIps(t *testing.T) { + tmpFileName, err := fileutil.GetTempFileName() + assert.Nil(t, err) + expectedIpsFromCLI := []string{"8.8.8.0/24", "7.7.7.7"} + expectedIpsFromFile := []string{"10.10.10.0/24", "192.168.1.0/24"} + assert.Nil(t, os.WriteFile(tmpFileName, []byte(strings.Join(expectedIpsFromFile, "\n")), 0755)) + expected := append(expectedIpsFromCLI, expectedIpsFromFile...) + + actual, err := parseExcludedIps(&Options{ + ExcludeIps: strings.Join(expectedIpsFromCLI, ","), + ExcludeIpsFile: tmpFileName, + }) + assert.Nil(t, err) + assert.Equal(t, expected, actual) + + defer os.RemoveAll(tmpFileName) +} + +func TestIsIpOrCidr(t *testing.T) { + valid := []string{"1.1.1.1", "2.2.2.2", "1.1.1.0/24"} + invalid := []string{"1.1.1.1.1", "a.a.a.a", "77"} + for _, validItem := range valid { + assert.True(t, isIpOrCidr(validItem)) + } + for _, invalidItem := range invalid { + assert.False(t, isIpOrCidr(invalidItem)) + } +} diff --git a/v2/pkg/runner/nmap.go b/v2/pkg/runner/nmap.go index fd5f2ce0..8dbf3cdc 100644 --- a/v2/pkg/runner/nmap.go +++ b/v2/pkg/runner/nmap.go @@ -6,10 +6,12 @@ import ( "os/exec" "strings" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) -func (r *Runner) handleNmap() { +func (r *Runner) handleNmap() error { // command from CLI command := r.options.NmapCLI hasCLI := r.options.NmapCLI != "" @@ -33,8 +35,9 @@ func (r *Runner) handleNmap() { // if we have no open ports we avoid running nmap if len(ports) == 0 { - gologger.Info().Msgf("Skipping nmap scan as no open ports were found") - return + errMsg := errors.New("Skipping nmap scan as no open ports were found") + gologger.Info().Msgf(errMsg.Error()) + return errMsg } portsStr := strings.Join(ports, ",") @@ -50,11 +53,14 @@ func (r *Runner) handleNmap() { cmd.Stdout = os.Stdout err := cmd.Run() if err != nil { - gologger.Error().Msgf("Could not run nmap command: %s\n", err) - return + errMsg := errors.Wrap(err, "Could not run nmap command") + gologger.Error().Msgf(errMsg.Error()) + return errMsg } } else { gologger.Info().Msgf("Suggested nmap command: %s -p %s %s", command, portsStr, ipsStr) } } + + return nil } diff --git a/v2/pkg/runner/nmap_test.go b/v2/pkg/runner/nmap_test.go new file mode 100644 index 00000000..92b54528 --- /dev/null +++ b/v2/pkg/runner/nmap_test.go @@ -0,0 +1,25 @@ +package runner + +import ( + "testing" + + "github.com/projectdiscovery/naabu/v2/pkg/result" + "github.com/projectdiscovery/naabu/v2/pkg/scan" + "github.com/stretchr/testify/assert" +) + +func TestHandleNmap(t *testing.T) { + // just attempt to start nmap + var r Runner + r.options = &Options{} + // nmap with empty cli shouldn't trigger any error + res := result.NewResult() + r.scanner = &scan.Scanner{} + r.scanner.ScanResults = res + assert.Nil(t, r.handleNmap()) + r.scanner.ScanResults.IPPorts = make(map[string]map[int]struct{}) + // nmap syntax error (this test might fail if nmap is not installed on the box) + assert.Nil(t, r.handleNmap()) + r.scanner.ScanResults.IPPorts = map[string]map[int]struct{}{"127.0.0.1": {8080: struct{}{}}} + assert.Nil(t, r.handleNmap()) +} diff --git a/v2/pkg/runner/output_test.go b/v2/pkg/runner/output_test.go new file mode 100644 index 00000000..45f7681a --- /dev/null +++ b/v2/pkg/runner/output_test.go @@ -0,0 +1,28 @@ +package runner + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWriteHostOutput(t *testing.T) { + host := "127.0.0.1" + ports := map[int]struct{}{80: {}, 8080: {}} + var s string + buf := bytes.NewBufferString(s) + assert.Nil(t, WriteHostOutput(host, ports, buf)) + assert.Equal(t, "127.0.0.1:80\n127.0.0.1:8080\n", buf.String()) +} + +func TestWriteJSONOutput(t *testing.T) { + host := "localhost" + ip := "127.0.0.1" + ports := map[int]struct{}{80: {}, 8080: {}} + var s string + buf := bytes.NewBufferString(s) + assert.Nil(t, WriteJSONOutput(host, ip, ports, buf)) + assert.Equal(t, 3, len(strings.Split(buf.String(), "\n"))) +} diff --git a/v2/pkg/runner/ports.go b/v2/pkg/runner/ports.go index d2dc7483..71e15d37 100644 --- a/v2/pkg/runner/ports.go +++ b/v2/pkg/runner/ports.go @@ -6,6 +6,8 @@ import ( "sort" "strconv" "strings" + + "github.com/pkg/errors" ) const portListStrParts = 2 @@ -42,8 +44,8 @@ func ParsePorts(options *Options) ([]int, error) { // If the user has specfied top ports, use them as well if options.TopPorts != "" { - // If the user has specfied full ports, use them - if strings.EqualFold(options.TopPorts, "full") { + switch strings.ToLower(options.TopPorts) { + case "full": // If the user has specfied full ports, use them var err error ports, err := parsePortsList(Full) if err != nil { @@ -53,10 +55,7 @@ func ParsePorts(options *Options) ([]int, error) { if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) } - } - - // If the user has specfied top-100, use them - if strings.EqualFold(options.TopPorts, "top-100") { + case "top-100": // If the user has specfied top-100, use them ports, err := parsePortsList(NmapTop100) if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) @@ -65,10 +64,7 @@ func ParsePorts(options *Options) ([]int, error) { if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) } - } - - // If the user has specfied top-1000, use them - if strings.EqualFold(options.TopPorts, "top-1000") { + case "top-1000": // If the user has specfied top-1000, use them ports, err := parsePortsList(NmapTop1000) if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) @@ -77,10 +73,12 @@ func ParsePorts(options *Options) ([]int, error) { if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) } + default: + return nil, errors.New("invalid top ports option") } } - // If the user has specfied top option, use them too + // If the user has specfied ports option, use them too if options.Ports != "" { // "-" equals to all ports if options.Ports == "-" { @@ -97,7 +95,7 @@ func ParsePorts(options *Options) ([]int, error) { } } - // merge all the specified ports (meaningless is "all" is used) + // merge all the specified ports (meaningless if "all" is used) portsConfigMap := merge(portsConfigList...) ports := merge(portsFileMap, portsCLIMap, topPortsCLIMap, portsConfigMap) diff --git a/v2/pkg/runner/ports_test.go b/v2/pkg/runner/ports_test.go new file mode 100644 index 00000000..2af1d0e4 --- /dev/null +++ b/v2/pkg/runner/ports_test.go @@ -0,0 +1,111 @@ +package runner + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParsePortsList(t *testing.T) { + tests := []struct { + args string + want map[int]struct{} + wantErr bool + }{ + {"1,2,3,4", map[int]struct{}{1: {}, 2: {}, 3: {}, 4: {}}, false}, + {"1-3,10", map[int]struct{}{1: {}, 2: {}, 3: {}, 10: {}}, false}, + {"17,17,17,18", map[int]struct{}{17: {}, 18: {}}, false}, + {"a", nil, true}, + } + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got, err := parsePortsList(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("parsePortsList() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parsePortsList() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExcludePorts(t *testing.T) { + var options Options + ports := map[int]struct{}{1: {}, 10: {}} + + // no filtering + filteredPorts, err := excludePorts(&options, ports) + assert.Nil(t, err) + assert.EqualValues(t, filteredPorts, ports) + + // invalida filter + options.ExcludePorts = "a" + _, err = excludePorts(&options, ports) + assert.NotNil(t, err) + + // valid filter + options.ExcludePorts = "1" + filteredPorts, err = excludePorts(&options, ports) + assert.Nil(t, err) + expectedPorts := map[int]struct{}{10: {}} + assert.EqualValues(t, expectedPorts, filteredPorts) +} + +func TestParsePorts(t *testing.T) { + // top ports + tests := []struct { + args string + want int + wantErr bool + }{ + {"full", 65535, false}, + {"top-100", 100, false}, + {"top-1000", 1000, false}, + {"a", 0, true}, + } + for _, tt := range tests { + t.Run(tt.args, func(t *testing.T) { + var options Options + options.TopPorts = tt.args + got, err := ParsePorts(&options) + if tt.wantErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, tt.want, len(got)) + }) + } + + // ports + tests = []struct { + args string + want int + wantErr bool + }{ + {"-", 65535, false}, + {"a", 0, true}, + {"1,2,4-10", 9, false}, + } + for _, tt := range tests { + t.Run(tt.args, func(t *testing.T) { + var options Options + options.Ports = tt.args + got, err := ParsePorts(&options) + if tt.wantErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, tt.want, len(got)) + }) + } + + // default to 100 ports + got, err := ParsePorts(&Options{}) + assert.Nil(t, err) + assert.Equal(t, 100, len(got)) +} diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 05f98590..bd97eb0e 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -194,9 +194,7 @@ func (r *Runner) RunEnumeration() error { r.handleOutput() // handle nmap - r.handleNmap() - - return nil + return r.handleNmap() } // Close runner instance diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go index 31693cc4..0573cfff 100644 --- a/v2/pkg/runner/util.go +++ b/v2/pkg/runner/util.go @@ -2,12 +2,11 @@ package runner import ( "fmt" - "net" "os" "runtime" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/mapcidr" + "github.com/projectdiscovery/iputil" ) func isRoot() bool { @@ -17,7 +16,7 @@ func isRoot() bool { func (r *Runner) host2ips(target string) (targetIPs []string, err error) { // If the host is a Domain, then perform resolution and discover all IP // addresses for a given host. Else use that host for port scanning - if net.ParseIP(target) == nil { + if !iputil.IsIP(target) { var ips []string ips, err = r.dnsclient.Lookup(target) if err != nil { @@ -25,7 +24,7 @@ func (r *Runner) host2ips(target string) (targetIPs []string, err error) { return } for _, ip := range ips { - if mapcidr.IsIPv4(net.ParseIP(ip)) { + if iputil.IsIPv4(ip) { targetIPs = append(targetIPs, ip) } } diff --git a/v2/pkg/runner/util_test.go b/v2/pkg/runner/util_test.go new file mode 100644 index 00000000..f80c0042 --- /dev/null +++ b/v2/pkg/runner/util_test.go @@ -0,0 +1,42 @@ +package runner + +import ( + "testing" + + "github.com/projectdiscovery/dnsx/libs/dnsx" + "github.com/stretchr/testify/assert" +) + +func Test_host2ips(t *testing.T) { + tests := []struct { + args string + want []string + wantErr bool + }{ + {"10.10.10.10", []string{"10.10.10.10"}, false}, + {"localhost", []string{"127.0.0.1"}, false}, + {"aaaa", nil, true}, + {"10.10.10.0/24", nil, true}, + } + + var r Runner + if dnsclient, err := dnsx.New(dnsx.DefaultOptions); err != nil { + assert.Error(t, err) + } else { + r.dnsclient = dnsclient + } + + for _, tt := range tests { + t.Run(tt.args, func(t *testing.T) { + var options Options + options.TopPorts = tt.args + got, err := r.host2ips(tt.args) + if tt.wantErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index d9ab5adf..9458bfad 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -1,39 +1,46 @@ package runner import ( - "errors" "flag" "fmt" "net" "strings" + "github.com/pkg/errors" "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" ) +var ( + errNoInputList = errors.New("no input list provided") + errOutputMode = errors.New("both verbose and silent mode specified") + errZeroValue = errors.New("cannot be zero") +) + // validateOptions validates the configuration options passed func (options *Options) validateOptions() error { // Check if Host, list of domains, or stdin info was provided. // If none was provided, then return. if options.Host == "" && options.HostsFile == "" && !options.Stdin && len(flag.Args()) == 0 { - return errors.New("no input list provided") + return errNoInputList } // Both verbose and silent flags were used if options.Verbose && options.Silent { - return errors.New("both verbose and silent mode specified") + return errOutputMode } if options.Timeout == 0 { - return errors.New("timeout cannot be zero") + return errors.Wrap(errZeroValue, "timeout") } else if !isRoot() && options.Timeout == DefaultPortTimeoutSynScan { options.Timeout = DefaultPortTimeoutConnectScan } if options.Rate == 0 { - return errors.New("rate cannot be zero") + return errors.Wrap(errZeroValue, "rate") } else if !isRoot() && options.Rate == DefaultRateSynScan { options.Rate = DefaultRateConnectScan } diff --git a/v2/pkg/runner/validate_test.go b/v2/pkg/runner/validate_test.go new file mode 100644 index 00000000..49f80297 --- /dev/null +++ b/v2/pkg/runner/validate_test.go @@ -0,0 +1,22 @@ +package runner + +import ( + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func TestOptions(t *testing.T) { + options := Options{} + assert.ErrorIs(t, errNoInputList, options.validateOptions()) + + options.Host = "target" + assert.EqualError(t, options.validateOptions(), errors.Wrap(errZeroValue, "timeout").Error()) + + options.Timeout = 2 + assert.EqualError(t, options.validateOptions(), errors.Wrap(errZeroValue, "rate").Error()) + + options.Resolvers = "aaabbbccc" + assert.NotNil(t, options.validateOptions()) +} From 35d458f716ee21220b9178a3715f0c53137091d6 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 29 Nov 2021 00:13:23 +0100 Subject: [PATCH 12/67] adding libpcap for codeql analysis --- .github/workflows/codeql-analysis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 545cdea9..9ea93430 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -22,6 +22,9 @@ jobs: # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] steps: + - name: Install libpcap-dev + run: sudo apt install libpcap-dev + - name: Checkout repository uses: actions/checkout@v2 From fee31882d525f66aceb7ab29586e1a538ce3fed9 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Sun, 28 Nov 2021 21:29:33 -0600 Subject: [PATCH 13/67] added simplehttpserver in workflow --- .github/workflows/functional-test.yml | 9 ++++++++ v2/cmd/functional-test/testcases.txt | 32 +++++++++++++-------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 427904ac..ce69bd5d 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -17,6 +17,15 @@ jobs: - name: Install libpcap-dev run: sudo apt install libpcap-dev + + - name: Install nmap + run: sudo apt install nmap + + - name: Install simplehttpserver + run: GO111MODULE=on go get -v github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver + + - name: Run the simplehttpserver on 127.0.0.1:8000 + run: simplehttpserver -listen 127.0.0.1:8000 - name: Check out code uses: actions/checkout@v2 diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index bc022b5c..f0809e7d 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,19 +1,19 @@ -example.com {{binary}} -silent -example.com {{binary}} -silent -il -example.com {{binary}} -silent l test-data/request.txt -example.com {{binary}} -silent -p 80,443 -example.com {{binary}} -silent -tp 100 -example.com {{binary}} -silent -ep 200 -example.com {{binary}} -silent -ec 100,400 -example.com {{binary}} -silent -version -example.com {{binary}} -silent -retries 5 -example.com {{binary}} -silent -c 25 -example.com {{binary}} -silent -verify -example.com {{binary}} -silent -timeout 30 -example.com {{binary}} -silent -r test-data/request.txt -example.com {{binary}} -silent -warm-up-time 2 -example.com {{binary}} -silent -nmap-cli 'nmap -sV' -example.com {{binary}} -silent -c 25 +127.0.0.1 {{binary}} -silent +127.0.0.1 {{binary}} -silent -il +127.0.0.1 {{binary}} -silent -l test-data/request.txt +127.0.0.1 {{binary}} -silent -p 80,443 +127.0.0.1 {{binary}} -silent -tp 100 +127.0.0.1 {{binary}} -silent -ep 200 +127.0.0.1 {{binary}} -silent -ec 100,400 +127.0.0.1 {{binary}} -silent -version +127.0.0.1 {{binary}} -silent -retries 5 +127.0.0.1 {{binary}} -silent -c 25 +127.0.0.1 {{binary}} -silent -verify +127.0.0.1 {{binary}} -silent -timeout 30 +127.0.0.1 {{binary}} -silent -r test-data/request.txt +127.0.0.1 {{binary}} -silent -warm-up-time 2 +127.0.0.1 {{binary}} -silent -nmap-cli 'nmap -sV' +127.0.0.1 {{binary}} -silent -c 25 www.projectdiscovery.io {{binary}} -silent -eh 127.0.0.1:8082,127.0.0.1:6089 www.projectdiscovery.io {{binary}} -silent -stats 1.1.1.1 {{binary}} -silent -json From 785b7adef36be12ba347ba170b681e46e8955756 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Sun, 28 Nov 2021 21:38:44 -0600 Subject: [PATCH 14/67] workflow run --- .github/workflows/functional-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index ce69bd5d..555c62fb 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -25,7 +25,7 @@ jobs: run: GO111MODULE=on go get -v github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver - name: Run the simplehttpserver on 127.0.0.1:8000 - run: simplehttpserver -listen 127.0.0.1:8000 + run: bash simplehttpserver -listen 127.0.0.1:8000 - name: Check out code uses: actions/checkout@v2 From d2edd3bb3b1ed19e71997ee9c837298ee6e326a1 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Sun, 28 Nov 2021 21:48:11 -0600 Subject: [PATCH 15/67] workflow update --- .github/workflows/functional-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 555c62fb..40f9902e 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -25,7 +25,9 @@ jobs: run: GO111MODULE=on go get -v github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver - name: Run the simplehttpserver on 127.0.0.1:8000 - run: bash simplehttpserver -listen 127.0.0.1:8000 + run: | + chmod +x simplehttpserver + bash simplehttpserver -listen 127.0.0.1:8000 - name: Check out code uses: actions/checkout@v2 From d126cd8e3b2f1612506fb131defc4e8f4a4d22be Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Sun, 28 Nov 2021 21:55:17 -0600 Subject: [PATCH 16/67] simplehttpserver addition --- .github/workflows/functional-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 40f9902e..b5430ecd 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -22,7 +22,7 @@ jobs: run: sudo apt install nmap - name: Install simplehttpserver - run: GO111MODULE=on go get -v github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver + run: go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest - name: Run the simplehttpserver on 127.0.0.1:8000 run: | From bb05af083b6393c4ae0d07508c1e6c31c56e0e55 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 29 Nov 2021 07:55:14 +0100 Subject: [PATCH 17/67] removing empty file --- v2/pkg/scan/scan_darwin.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 v2/pkg/scan/scan_darwin.go diff --git a/v2/pkg/scan/scan_darwin.go b/v2/pkg/scan/scan_darwin.go deleted file mode 100644 index 040485ba..00000000 --- a/v2/pkg/scan/scan_darwin.go +++ /dev/null @@ -1 +0,0 @@ -package scan From 21c6f0db2df7d006c8ea09c71926ea22c29cd48e Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 29 Nov 2021 07:55:28 +0100 Subject: [PATCH 18/67] adding remaining package tests --- v2/pkg/scan/cdn_test.go | 32 ++++++++++++++++++++++++ v2/pkg/scan/connect_test.go | 32 ++++++++++++++++++++++++ v2/pkg/scan/externalip_test.go | 13 ++++++++++ v2/pkg/scan/ping_test.go | 42 ++++++++++++++++++++++++++++++++ v2/pkg/scan/tcpsequencer_test.go | 16 ++++++++++++ 5 files changed, 135 insertions(+) create mode 100644 v2/pkg/scan/cdn_test.go create mode 100644 v2/pkg/scan/connect_test.go create mode 100644 v2/pkg/scan/externalip_test.go create mode 100644 v2/pkg/scan/ping_test.go create mode 100644 v2/pkg/scan/tcpsequencer_test.go diff --git a/v2/pkg/scan/cdn_test.go b/v2/pkg/scan/cdn_test.go new file mode 100644 index 00000000..02272ab6 --- /dev/null +++ b/v2/pkg/scan/cdn_test.go @@ -0,0 +1,32 @@ +package scan + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCdnCheck(t *testing.T) { + s, err := NewScanner(&Options{ExcludeCdn: true}) + assert.Nil(t, err) + tests := []struct { + args string + want bool + wantErr bool + }{ + {"192.168.1.1", false, false}, + {"10.10.10.10", false, false}, + {"aaaaa", false, true}, + } + for _, tt := range tests { + t.Run(tt.args, func(t *testing.T) { + isCdn, err := s.CdnCheck(tt.args) + if tt.wantErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, tt.want, isCdn) + }) + } +} diff --git a/v2/pkg/scan/connect_test.go b/v2/pkg/scan/connect_test.go new file mode 100644 index 00000000..72224dda --- /dev/null +++ b/v2/pkg/scan/connect_test.go @@ -0,0 +1,32 @@ +package scan + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConnectVerify(t *testing.T) { + go func() { + // start tcp server + l, err := net.Listen("tcp", ":17895") + if err != nil { + assert.Nil(t, err) + } + defer l.Close() + for { + conn, err := l.Accept() + if err != nil { + return + } + defer conn.Close() + } + }() + + s, err := NewScanner(&Options{}) + assert.Nil(t, err) + wanted := map[int]struct{}{17895: {}} + got := s.ConnectVerify("localhost", map[int]struct{}{17895: {}, 17896: {}}) + assert.EqualValues(t, wanted, got) +} diff --git a/v2/pkg/scan/externalip_test.go b/v2/pkg/scan/externalip_test.go new file mode 100644 index 00000000..43ca9a65 --- /dev/null +++ b/v2/pkg/scan/externalip_test.go @@ -0,0 +1,13 @@ +package scan + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWhatsMyIP(t *testing.T) { + externalIp, err := WhatsMyIP() + assert.Nil(t, err) + assert.NotEmpty(t, externalIp) +} diff --git a/v2/pkg/scan/ping_test.go b/v2/pkg/scan/ping_test.go new file mode 100644 index 00000000..209e3f46 --- /dev/null +++ b/v2/pkg/scan/ping_test.go @@ -0,0 +1,42 @@ +package scan + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPingHosts(t *testing.T) { + if os.Getuid() == 0 { + tests := []struct { + args string + want bool + wantErr bool + }{ + {"127.0.0.1", true, false}, + {"localhost", true, false}, + {"aaaaa", false, true}, + } + + for _, tt := range tests { + t.Run(tt.args, func(t *testing.T) { + pingResults, err := PingHosts([]string{tt.args}) + if tt.wantErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + if tt.want { + assert.NotEmpty(t, pingResults) + fastest, err := pingResults.GetFastestHost() + assert.Nil(t, err) + assert.NotEmpty(t, pingResults.Hosts) + assert.NotEmpty(t, fastest.Host) + } else { + assert.Empty(t, pingResults) + } + }) + } + } +} diff --git a/v2/pkg/scan/tcpsequencer_test.go b/v2/pkg/scan/tcpsequencer_test.go new file mode 100644 index 00000000..2ebede52 --- /dev/null +++ b/v2/pkg/scan/tcpsequencer_test.go @@ -0,0 +1,16 @@ +package scan + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTCPSequencer(t *testing.T) { + tcpSequencer := NewTCPSequencer() + // tcp sequencer should be uint32 incremental + for i := 0; i < 50000; i++ { + actual := tcpSequencer.Next() + assert.Equal(t, uint32(i), actual) + } +} From c1a12314c01b1b156fb16240d22f0b4e222293ed Mon Sep 17 00:00:00 2001 From: Passer6y <34622501+Passer6y@users.noreply.github.com> Date: Mon, 29 Nov 2021 16:09:55 +0800 Subject: [PATCH 19/67] Update targets.go --- v2/pkg/runner/targets.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/v2/pkg/runner/targets.go b/v2/pkg/runner/targets.go index 3e6d7521..ad950f15 100644 --- a/v2/pkg/runner/targets.go +++ b/v2/pkg/runner/targets.go @@ -44,14 +44,9 @@ func (r *Runner) mergeToFile() (string, error) { // target defined via CLI argument if r.options.Host != "" { - if strings.Contains(r.Options.Host, ","){ - splitHosts := strings.Split(r.Options.Host, ",") - for _, host := range splitHosts{ - fmt.Fprintf(tempInput, "%s\n", host) - } - }else{ - fmt.Fprintf(tempInput, "%s\n", r.Options.Host) - } + for _, host := range strings.Split(r.options.Host, ",") { + fmt.Fprintf(tempInput, "%s\n", host) + } } // Targets from file From 173a60e6548e44d99b347b2bf878a90fb4db02e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 13:04:43 +0000 Subject: [PATCH 20/67] chore(deps): bump alpine from 3.14 to 3.15.0 Bumps alpine from 3.14 to 3.15.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 526f48f4..edf95760 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.3-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest -FROM alpine:3.14 +FROM alpine:3.15.0 RUN apk add nmap libpcap-dev bind-tools ca-certificates COPY --from=builder /go/bin/naabu /usr/local/bin/naabu ENTRYPOINT ["naabu"] From a2fa517ea04d33db15b995c14318e17d9894e211 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 30 Nov 2021 13:12:50 +0100 Subject: [PATCH 21/67] removing log.fatal from internal code --- v2/pkg/scan/scan.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/v2/pkg/scan/scan.go b/v2/pkg/scan/scan.go index 3cac29af..bd84ebcf 100644 --- a/v2/pkg/scan/scan.go +++ b/v2/pkg/scan/scan.go @@ -2,7 +2,6 @@ package scan import ( "fmt" - "log" "math/rand" "net" "strings" @@ -149,7 +148,7 @@ func NewScanner(options *Options) (*Scanner, error) { if options.Proxy != "" { proxyDialer, err := proxy.SOCKS5("tcp", options.Proxy, nil, &net.Dialer{Timeout: options.Timeout}) if err != nil { - log.Fatal(err) + return nil, err } scanner.proxyDialer = proxyDialer } @@ -375,7 +374,7 @@ func (s *Scanner) ConnectPort(host string, port int, timeout time.Duration) (boo if s.proxyDialer != nil { conn, err = s.proxyDialer.Dial("tcp", hostport) if err != nil { - log.Fatal(err) + return false, err } } else { conn, err = net.DialTimeout("tcp", hostport, timeout) From e7fc842ea0b237bb5236b81e9a189c2db3e29372 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 30 Nov 2021 13:50:28 -0600 Subject: [PATCH 22/67] added simplehttpserver in box setup and modified testcases --- .github/workflows/functional-test.yml | 28 ++++++++++----- v2/cmd/functional-test/test-data/request.txt | 2 +- v2/cmd/functional-test/testcases.txt | 36 ++++++++++---------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index b5430ecd..1dbe2438 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -6,6 +6,26 @@ on: jobs: + setup-simplehttpserver: + name: Setup Simplehttpserver + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Install simplehttpserver + run: go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest + + - name: Install coreutils/gtimeout + run: sudo apt install coreutils + + - name: Run the simplehttpserver on 127.0.0.1:8000 + run: | + export PATH=$PATH:$(go env GOPATH)/bin + timeout -s SIGTERM -k 3m 3m simplehttpserver -listen 127.0.0.1:8000 & + functional: name: Functional Test runs-on: ubuntu-latest @@ -21,14 +41,6 @@ jobs: - name: Install nmap run: sudo apt install nmap - - name: Install simplehttpserver - run: go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest - - - name: Run the simplehttpserver on 127.0.0.1:8000 - run: | - chmod +x simplehttpserver - bash simplehttpserver -listen 127.0.0.1:8000 - - name: Check out code uses: actions/checkout@v2 diff --git a/v2/cmd/functional-test/test-data/request.txt b/v2/cmd/functional-test/test-data/request.txt index de54ac66..89ff1b36 100644 --- a/v2/cmd/functional-test/test-data/request.txt +++ b/v2/cmd/functional-test/test-data/request.txt @@ -1 +1 @@ -example.com +127.0.0.1 diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index f0809e7d..6846214b 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,23 +1,23 @@ -127.0.0.1 {{binary}} -silent +127.0.0.1 {{binary}} -silent -p 8000 127.0.0.1 {{binary}} -silent -il -127.0.0.1 {{binary}} -silent -l test-data/request.txt -127.0.0.1 {{binary}} -silent -p 80,443 -127.0.0.1 {{binary}} -silent -tp 100 -127.0.0.1 {{binary}} -silent -ep 200 -127.0.0.1 {{binary}} -silent -ec 100,400 +127.0.0.1 {{binary}} -silent -l test-data/request.txt -p 8000 +127.0.0.1 {{binary}} -silent -p 8000,443 +127.0.0.1 {{binary}} -silent -tp 100 -p 8000 +127.0.0.1 {{binary}} -silent -ep 200 -p 8000 +127.0.0.1 {{binary}} -silent -ec 100,400 -p 8000 127.0.0.1 {{binary}} -silent -version -127.0.0.1 {{binary}} -silent -retries 5 -127.0.0.1 {{binary}} -silent -c 25 -127.0.0.1 {{binary}} -silent -verify -127.0.0.1 {{binary}} -silent -timeout 30 -127.0.0.1 {{binary}} -silent -r test-data/request.txt -127.0.0.1 {{binary}} -silent -warm-up-time 2 -127.0.0.1 {{binary}} -silent -nmap-cli 'nmap -sV' -127.0.0.1 {{binary}} -silent -c 25 -www.projectdiscovery.io {{binary}} -silent -eh 127.0.0.1:8082,127.0.0.1:6089 -www.projectdiscovery.io {{binary}} -silent -stats -1.1.1.1 {{binary}} -silent -json -1.1.1.1 {{binary}} -silent -s CONNECT +127.0.0.1 {{binary}} -silent -retries 5 -p 8000 +127.0.0.1 {{binary}} -silent -c 25 -p 8000 +127.0.0.1 {{binary}} -silent -verify -p 8000 +127.0.0.1 {{binary}} -silent -timeout 30 -p 8000 +127.0.0.1 {{binary}} -silent -r test-data/request.txt -p 8000 +127.0.0.1 {{binary}} -silent -warm-up-time 2 -p 8000 +127.0.0.1 {{binary}} -silent -nmap-cli 'nmap -sV' -p 8000 +127.0.0.1 {{binary}} -silent -eh 127.0.0.1:8082,127.0.0.1:6089 +127.0.0.1 {{binary}} -silent -stats +127.0.0.1 {{binary}} -silent -json +127.0.0.1 {{binary}} -silent -s CONNECT + From e9f3368c6e0acd4d9b987c12b5bafc055913d3f8 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 30 Nov 2021 17:17:15 -0600 Subject: [PATCH 23/67] removed silent flag and extra spaces --- v2/cmd/functional-test/testcases.txt | 41 ++++++++++++---------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 6846214b..55f7ab01 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,24 +1,19 @@ -127.0.0.1 {{binary}} -silent -p 8000 +127.0.0.1 {{binary}} -p 8000 127.0.0.1 {{binary}} -silent -il -127.0.0.1 {{binary}} -silent -l test-data/request.txt -p 8000 -127.0.0.1 {{binary}} -silent -p 8000,443 -127.0.0.1 {{binary}} -silent -tp 100 -p 8000 -127.0.0.1 {{binary}} -silent -ep 200 -p 8000 -127.0.0.1 {{binary}} -silent -ec 100,400 -p 8000 -127.0.0.1 {{binary}} -silent -version -127.0.0.1 {{binary}} -silent -retries 5 -p 8000 -127.0.0.1 {{binary}} -silent -c 25 -p 8000 -127.0.0.1 {{binary}} -silent -verify -p 8000 -127.0.0.1 {{binary}} -silent -timeout 30 -p 8000 -127.0.0.1 {{binary}} -silent -r test-data/request.txt -p 8000 -127.0.0.1 {{binary}} -silent -warm-up-time 2 -p 8000 -127.0.0.1 {{binary}} -silent -nmap-cli 'nmap -sV' -p 8000 -127.0.0.1 {{binary}} -silent -eh 127.0.0.1:8082,127.0.0.1:6089 -127.0.0.1 {{binary}} -silent -stats -127.0.0.1 {{binary}} -silent -json -127.0.0.1 {{binary}} -silent -s CONNECT - - - - - +127.0.0.1 {{binary}} -l test-data/request.txt -p 8000 +127.0.0.1 {{binary}} -p 8000,443 +127.0.0.1 {{binary}} -tp 100 -p 8000 +127.0.0.1 {{binary}} -ep 200 -p 8000 +127.0.0.1 {{binary}} -ec 100,400 -p 8000 +127.0.0.1 {{binary}} -version +127.0.0.1 {{binary}} -retries 5 -p 8000 +127.0.0.1 {{binary}} -c 25 -p 8000 +127.0.0.1 {{binary}} -verify -p 8000 +127.0.0.1 {{binary}} -timeout 30 -p 8000 +127.0.0.1 {{binary}} -r test-data/request.txt -p 8000 +127.0.0.1 {{binary}} -warm-up-time 2 -p 8000 +127.0.0.1 {{binary}} -nmap-cli 'nmap -sV' -p 8000 +127.0.0.1 {{binary}} -eh 127.0.0.1:8082,127.0.0.1:6089 +127.0.0.1 {{binary}} -stats +127.0.0.1 {{binary}} -json +127.0.0.1 {{binary}} -s CONNECT From c4562bde7f42056024c1fe995167fd783df3e247 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 30 Nov 2021 20:18:02 -0600 Subject: [PATCH 24/67] added simplehttpserver installation and run as a step in functional test job --- .github/workflows/functional-test.yml | 28 ++++++++------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 1dbe2438..583323ce 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -6,26 +6,6 @@ on: jobs: - setup-simplehttpserver: - name: Setup Simplehttpserver - runs-on: ubuntu-latest - steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: Install simplehttpserver - run: go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest - - - name: Install coreutils/gtimeout - run: sudo apt install coreutils - - - name: Run the simplehttpserver on 127.0.0.1:8000 - run: | - export PATH=$PATH:$(go env GOPATH)/bin - timeout -s SIGTERM -k 3m 3m simplehttpserver -listen 127.0.0.1:8000 & - functional: name: Functional Test runs-on: ubuntu-latest @@ -40,10 +20,18 @@ jobs: - name: Install nmap run: sudo apt install nmap + + - name: Install simplehttpserver + run: go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest - name: Check out code uses: actions/checkout@v2 + - name: Run the simplehttpserver on 127.0.0.1:8000 + run: | + export PATH=$PATH:$(go env GOPATH)/bin + simplehttpserver -listen 127.0.0.1:8000 & + - name: Functional Tests run: | chmod +x run.sh From 91b3142eebf03f9383ff4b743f1425080ba2c553 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 1 Dec 2021 09:45:54 +0100 Subject: [PATCH 25/67] nmap tuning - Skip host discovery - Full connect scan --- v2/cmd/functional-test/testcases.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 55f7ab01..61670933 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -12,7 +12,7 @@ 127.0.0.1 {{binary}} -timeout 30 -p 8000 127.0.0.1 {{binary}} -r test-data/request.txt -p 8000 127.0.0.1 {{binary}} -warm-up-time 2 -p 8000 -127.0.0.1 {{binary}} -nmap-cli 'nmap -sV' -p 8000 +127.0.0.1 {{binary}} -nmap-cli 'nmap -Pn -sT' -p 8000 127.0.0.1 {{binary}} -eh 127.0.0.1:8082,127.0.0.1:6089 127.0.0.1 {{binary}} -stats 127.0.0.1 {{binary}} -json From 8e5bef9aeebbb0f0da57144f4b16924ee17d8203 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 1 Dec 2021 09:52:01 +0100 Subject: [PATCH 26/67] using go install syntax --- v2/cmd/functional-test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 9527cb30..9b9b05df 100755 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -7,7 +7,7 @@ go build -o naabu_dev ../naabu echo 'Installing latest release of NAABU' - GO111MODULE=on go build -v github.com/projectdiscovery/naabu/v2/cmd/naabu +GO111MODULE=on go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu echo 'Starting NAABU functional test' ./functional-test -main ./naabu -dev ./naabu_dev -testcases testcases.txt From 97a7f6766006d3e45e7bfbd41ed4daf8ed821c0a Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 1 Dec 2021 10:01:47 +0100 Subject: [PATCH 27/67] removing extra spaces --- v2/cmd/functional-test/run.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 9b9b05df..70dba573 100755 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -1,13 +1,13 @@ #!/bin/bash - echo 'Building functional-test binary' - go build +echo 'Building functional-test binary' +go build - echo 'Building NAABU binary from current branch' - go build -o naabu_dev ../naabu +echo 'Building NAABU binary from current branch' +go build -o naabu_dev ../naabu - echo 'Installing latest release of NAABU' -GO111MODULE=on go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu +echo 'Installing latest release of NAABU' +GO111MODULE=on go build -v github.com/projectdiscovery/naabu/v2/cmd/naabu - echo 'Starting NAABU functional test' - ./functional-test -main ./naabu -dev ./naabu_dev -testcases testcases.txt +echo 'Starting NAABU functional test' +./functional-test -main ./naabu -dev ./naabu_dev -testcases testcases.txt From 99ad8cb5bf9db4a705c8b326a8daf28c510b3126 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 2 Dec 2021 11:14:42 +0530 Subject: [PATCH 28/67] test case update --- v2/cmd/functional-test/testcases.txt | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 61670933..2c735603 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,19 +1,7 @@ 127.0.0.1 {{binary}} -p 8000 -127.0.0.1 {{binary}} -silent -il -127.0.0.1 {{binary}} -l test-data/request.txt -p 8000 127.0.0.1 {{binary}} -p 8000,443 -127.0.0.1 {{binary}} -tp 100 -p 8000 -127.0.0.1 {{binary}} -ep 200 -p 8000 -127.0.0.1 {{binary}} -ec 100,400 -p 8000 -127.0.0.1 {{binary}} -version -127.0.0.1 {{binary}} -retries 5 -p 8000 +127.0.0.1 {{binary}} -tp 100 +127.0.0.1 {{binary}} -ep 80 -p 8000 127.0.0.1 {{binary}} -c 25 -p 8000 -127.0.0.1 {{binary}} -verify -p 8000 -127.0.0.1 {{binary}} -timeout 30 -p 8000 -127.0.0.1 {{binary}} -r test-data/request.txt -p 8000 -127.0.0.1 {{binary}} -warm-up-time 2 -p 8000 127.0.0.1 {{binary}} -nmap-cli 'nmap -Pn -sT' -p 8000 -127.0.0.1 {{binary}} -eh 127.0.0.1:8082,127.0.0.1:6089 -127.0.0.1 {{binary}} -stats -127.0.0.1 {{binary}} -json -127.0.0.1 {{binary}} -s CONNECT +127.0.0.1 {{binary}} -json \ No newline at end of file From 1bdf58458416bb0ec381f0e6c3a4911f30f72c22 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 2 Dec 2021 17:25:02 +0530 Subject: [PATCH 29/67] removing comments --- .github/workflows/build-test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ed205d66..6b63bc66 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -28,10 +28,6 @@ jobs: run: go test ./... working-directory: v2/ - # - name: Integration Tests - # run: go run . - # working-directory: v2/cmd/integration_tests/ - - name: Build run: go build . working-directory: v2/cmd/naabu/ From 4aa96461384500e080907ff470cb18c6d204ead0 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 2 Dec 2021 17:36:01 +0530 Subject: [PATCH 30/67] fix: fixed top ports test case example --- v2/cmd/functional-test/testcases.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 2c735603..22f83fb0 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,6 +1,6 @@ 127.0.0.1 {{binary}} -p 8000 127.0.0.1 {{binary}} -p 8000,443 -127.0.0.1 {{binary}} -tp 100 +127.0.0.1 {{binary}} -tp top-100 127.0.0.1 {{binary}} -ep 80 -p 8000 127.0.0.1 {{binary}} -c 25 -p 8000 127.0.0.1 {{binary}} -nmap-cli 'nmap -Pn -sT' -p 8000 From c8750dfbeb15691634600904928d670c1baf1438 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 2 Dec 2021 17:46:48 +0530 Subject: [PATCH 31/67] fix: top ports options --- v2/cmd/functional-test/testcases.txt | 2 +- v2/pkg/runner/ports.go | 4 ++-- v2/pkg/runner/ports_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt index 22f83fb0..2c735603 100644 --- a/v2/cmd/functional-test/testcases.txt +++ b/v2/cmd/functional-test/testcases.txt @@ -1,6 +1,6 @@ 127.0.0.1 {{binary}} -p 8000 127.0.0.1 {{binary}} -p 8000,443 -127.0.0.1 {{binary}} -tp top-100 +127.0.0.1 {{binary}} -tp 100 127.0.0.1 {{binary}} -ep 80 -p 8000 127.0.0.1 {{binary}} -c 25 -p 8000 127.0.0.1 {{binary}} -nmap-cli 'nmap -Pn -sT' -p 8000 diff --git a/v2/pkg/runner/ports.go b/v2/pkg/runner/ports.go index 71e15d37..108ea190 100644 --- a/v2/pkg/runner/ports.go +++ b/v2/pkg/runner/ports.go @@ -55,7 +55,7 @@ func ParsePorts(options *Options) ([]int, error) { if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) } - case "top-100": // If the user has specfied top-100, use them + case "100": // If the user has specfied 100, use them ports, err := parsePortsList(NmapTop100) if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) @@ -64,7 +64,7 @@ func ParsePorts(options *Options) ([]int, error) { if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) } - case "top-1000": // If the user has specfied top-1000, use them + case "1000": // If the user has specfied 1000, use them ports, err := parsePortsList(NmapTop1000) if err != nil { return nil, fmt.Errorf("could not read ports: %s", err) diff --git a/v2/pkg/runner/ports_test.go b/v2/pkg/runner/ports_test.go index 2af1d0e4..207f9602 100644 --- a/v2/pkg/runner/ports_test.go +++ b/v2/pkg/runner/ports_test.go @@ -62,8 +62,8 @@ func TestParsePorts(t *testing.T) { wantErr bool }{ {"full", 65535, false}, - {"top-100", 100, false}, - {"top-1000", 1000, false}, + {"100", 100, false}, + {"1000", 1000, false}, {"a", 0, true}, } for _, tt := range tests { From 913f347ed1964d5a94e4cffbf65d2dd22010d2d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 13:04:45 +0000 Subject: [PATCH 32/67] chore(deps): bump golang from 1.17.3-alpine to 1.17.4-alpine Bumps golang from 1.17.3-alpine to 1.17.4-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index edf95760..44f6437c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.3-alpine AS builder +FROM golang:1.17.4-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From 8f68b94d6e134e20275143fa06f94f1aac422b08 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 6 Dec 2021 13:37:50 -0600 Subject: [PATCH 33/67] support for comma separated hosts --- v2/pkg/runner/options.go | 4 ++-- v2/pkg/runner/targets.go | 6 ++++-- v2/pkg/runner/validate.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 7a27743e..1a40a683 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -28,7 +28,7 @@ type Options struct { Rate int // Rate is the rate of port scan requests Timeout int // Timeout is the seconds to wait for ports to respond WarmUpTime int // WarmUpTime between scan phases - Host string // Host is the host to find ports for + Host goflags.NormalizedStringSlice // Host is the single host or comma-separated list of hosts to find ports for HostsFile string // HostsFile is the file containing list of hosts to find port for Output string // Output is the file to write found ports to. Ports string // Ports is the ports to use for enumeration @@ -62,7 +62,7 @@ func ParseOptions() *Options { flagSet.SetDescription(`Naabu is a port scanning tool written in Go that allows you to enumerate open ports for hosts in a fast and reliable manner.`) createGroup(flagSet, "input", "Input", - flagSet.StringVar(&options.Host, "host", "", "Host to scan ports for"), + flagSet.NormalizedStringSliceVarP(&options.Host, "host", "",[]string{}, "Single Host or comma-separated list of hosts to scan ports for"), flagSet.StringVarP(&options.HostsFile, "l", "list", "", "File containing list of hosts to scan ports"), flagSet.StringVarP(&options.ExcludeIps, "eh", "exclude-hosts", "", "Specifies a comma-separated list of targets to be excluded from the scan (ip, cidr)"), flagSet.StringVarP(&options.ExcludeIpsFile, "ef", "exclude-file", "", "Specifies a newline-delimited file with targets to be excluded from the scan (ip, cidr)"), diff --git a/v2/pkg/runner/targets.go b/v2/pkg/runner/targets.go index 9238f1f8..1f3a76ad 100644 --- a/v2/pkg/runner/targets.go +++ b/v2/pkg/runner/targets.go @@ -43,8 +43,10 @@ func (r *Runner) mergeToFile() (string, error) { defer tempInput.Close() // target defined via CLI argument - if r.options.Host != "" { - fmt.Fprintf(tempInput, "%s\n", r.options.Host) + if len(r.options.Host) > 0 { + for _,v :=range r.options.Host { + fmt.Fprintf(tempInput, "%s\n", v) + } } // Targets from file diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index 9458bfad..2f1d12c1 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -24,7 +24,7 @@ var ( func (options *Options) validateOptions() error { // Check if Host, list of domains, or stdin info was provided. // If none was provided, then return. - if options.Host == "" && options.HostsFile == "" && !options.Stdin && len(flag.Args()) == 0 { + if options.Host == nil && options.HostsFile == "" && !options.Stdin && len(flag.Args()) == 0 { return errNoInputList } From 1b934c592b9e6951bf0877cbd502d4bcea9306b9 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 6 Dec 2021 13:52:42 -0600 Subject: [PATCH 34/67] test case fix --- v2/pkg/runner/validate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/validate_test.go b/v2/pkg/runner/validate_test.go index 49f80297..d39834cd 100644 --- a/v2/pkg/runner/validate_test.go +++ b/v2/pkg/runner/validate_test.go @@ -11,7 +11,7 @@ func TestOptions(t *testing.T) { options := Options{} assert.ErrorIs(t, errNoInputList, options.validateOptions()) - options.Host = "target" + options.Host = []string{"target1", "target2"} assert.EqualError(t, options.validateOptions(), errors.Wrap(errZeroValue, "timeout").Error()) options.Timeout = 2 From 0401bd73c9a4bae95586105711d99211b3714a03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 13:05:42 +0000 Subject: [PATCH 35/67] chore(deps): bump golang from 1.17.4-alpine to 1.17.5-alpine Bumps golang from 1.17.4-alpine to 1.17.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 44f6437c..7882ca49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.4-alpine AS builder +FROM golang:1.17.5-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From 395a94e2276ea494edd7e45f1c4267ff9f6b1dee Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 9 Jan 2022 14:10:22 +0530 Subject: [PATCH 36/67] readme update --- README.md | 92 ++++++++++++++++----------------- v2/pkg/runner/options.go | 108 +++++++++++++++++++-------------------- 2 files changed, 97 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 1c8fa9c7..93ba3c60 100644 --- a/README.md +++ b/README.md @@ -33,13 +33,12 @@ all ports that return a reply.
- - Fast And Simple SYN/CONNECT probe based scanning. + - Fast And Simple **SYN/CONNECT** probe based scanning - Optimized for ease of use and **lightweight** on resources - - **Automatic handling of duplicate hosts between multiple subdomains** - - NMAP Integration for service discovery - - Piped input / output support for integrating in workflows - - Multiple Output formats supported (JSON, File, Stdout) - - Multiple input support including HOST/IP/CIDR notation. + - **Automatic IP deduplication for port scan** + - **NMAP** integration for service discovery + - Multiple input support - **STDIN/HOST/IP/CIDR** + - Multiple output format support - **JSON/TXT/STDOUT** # Usage @@ -49,54 +48,56 @@ naabu -h This will display help for the tool. Here are all the switches it supports. -```console +```yaml Usage: ./naabu [flags] INPUT: - -host string Host to scan ports for - -list, -l string File containing list of hosts to scan ports - -exclude-hosts, -eh string Specifies a comma-separated list of targets to be excluded from the scan (ip, cidr) - -exclude-file, -ef string Specifies a newline-delimited file with targets to be excluded from the scan (ip, cidr) + -host string[] hosts to scan ports for (comma-separated) + -list, -l string list of hosts to scan ports (file) + -exclude-hosts, -eh string hosts to exclude from the scan (comma-separated) + -exclude-file, -ef string list of hosts to exclude from scan (file) PORT: - -port, -p string Ports to scan (80, 80,443, 100-200 - -top-ports, -tp string Top Ports to scan (default top 100) - -exclude-ports, -ep string Ports to exclude from scan - -ports-file, -pf string File containing ports to scan for - -exclude-cdn, -ec Skip full port scans for CDNs (only checks for 80,443) + -port, -p string ports to scan (80,443, 100-200 + -top-ports, -tp string top ports to scan (default 100) + -exclude-ports, -ep string ports to exclude from scan (comma-separated) + -ports-file, -pf string list of ports to exclude from scan (file) + -exclude-cdn, -ec skip full port scans for CDN's (only checks for 80,443) RATE-LIMIT: - -c int General internal worker threads (default 25) - -rate int Rate of port scan probe request (default 1000) + -c int general internal worker threads (default 25) + -rate int packets to send per second (default 1000) OUTPUT: - -o, -output string File to write output to (optional) - -json Write output in JSON lines Format + -o, -output string file to write output to (optional) + -json write output in JSON lines format CONFIGURATION: - -scan-all-ips Scan all the ips - -scan-type, -s string Port scan type (SYN/CONNECT) (default s) - -source-ip string Source Ip - -interface-list, -il List available interfaces and public ip - -interface, -i string Network Interface to use for port scan - -nmap Invoke nmap scan on targets (nmap must be installed) + -scan-all-ips, -sa scan all the IP's associated with DNS record + -scan-type, -s string type of port scan (SYN/CONNECT) (default "s") + -source-ip string source ip + -interface-list, -il list available interfaces and public ip + -interface, -i string network Interface to use for port scan + -nmap invoke nmap scan on targets (nmap must be installed) - Deprecated -nmap-cli string nmap command to run on found results (example: -nmap-cli 'nmap -sV') + -r string list of custom resolver dns resolution (comma separated or from file) + -proxy string socks5 proxy OPTIMIZATION: - -retries int Number of retries for the port scan probe (default 3) - -timeout int Millisecond to wait before timing out (default 1000) - -warm-up-time int Time in seconds between scan phases (default 2) - -ping Use ping probes for verification of host - -verify Validate the ports again with TCP verification + -retries int number of retries for the port scan (default 3) + -timeout int millisecond to wait before timing out (default 1000) + -warm-up-time int time in seconds between scan phases (default 2) + -ping ping probes for verification of host + -verify validate the ports again with TCP verification DEBUG: - -debug Enable debugging information - -v Show Verbose output - -no-color, -nc Don't Use colors in output - -silent Show found ports only in output - -version Show version of naabu - -stats Display stats of the running scan + -debug display debugging information + -verbose, -v display verbose output + -no-color, -nc disable colors in CLI output + -silent display only results in output + -version display version of naabu + -stats display stats of the running scan ``` # Installation Instructions @@ -138,6 +139,7 @@ naabu -host hackerone.com [WRN] Developers assume no liability and are not responsible for any misuse or damage. [INF] Running SYN scan with root privileges [INF] Found 4 ports on host hackerone.com (104.16.100.52) + hackerone.com:80 hackerone.com:443 hackerone.com:8443 @@ -152,7 +154,7 @@ naabu -p 80,443,21-23 -host hackerone.com By default, the Naabu checks for nmap's `Top 100` ports. It supports following in-built port lists - -| CMD | Description | +| Flag | Description | |-------------------|--------------------------------------| | `-top-ports 100` | Scan for nmap top **100** port | | `-top-ports 1000` | Scan for nmap top **1000** port | @@ -164,12 +166,6 @@ You can also specify specific ports which you would like to exclude from the sca naabu -p - -exclude-ports 80,443 ``` -The `o` flag can be used to specify an output file. - -```sh -naabu -host hackerone.com -o output.txt -``` - To run the naabu on a list of hosts, `-list` option can be used. ```sh @@ -179,12 +175,10 @@ naabu -list hosts.txt You can also get output in json format using `-json` switch. This switch saves the output in the JSON lines format. ```console -naabu -host hackerone.com -json +naabu -host 104.16.99.52 -json -{"host":"hackerone.com","ip":"104.16.99.52","port":8443} -{"host":"hackerone.com","ip":"104.16.99.52","port":80} -{"host":"hackerone.com","ip":"104.16.99.52","port":443} -{"host":"hackerone.com","ip":"104.16.99.52","port":8080} +{"ip":"104.16.99.52","port":443} +{"ip":"104.16.99.52","port":80} ``` The ports discovered can be piped to other tools too. For example, you can pipe the ports discovered by naabu to [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host. diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 1a40a683..34dcce98 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -24,29 +24,29 @@ type Options struct { Nmap bool // Invoke nmap detailed scan on results InterfacesList bool // InterfacesList show interfaces list - Retries int // Retries is the number of retries for the port - Rate int // Rate is the rate of port scan requests - Timeout int // Timeout is the seconds to wait for ports to respond - WarmUpTime int // WarmUpTime between scan phases + Retries int // Retries is the number of retries for the port + Rate int // Rate is the rate of port scan requests + Timeout int // Timeout is the seconds to wait for ports to respond + WarmUpTime int // WarmUpTime between scan phases Host goflags.NormalizedStringSlice // Host is the single host or comma-separated list of hosts to find ports for - HostsFile string // HostsFile is the file containing list of hosts to find port for - Output string // Output is the file to write found ports to. - Ports string // Ports is the ports to use for enumeration - PortsFile string // PortsFile is the file containing ports to use for enumeration - ExcludePorts string // ExcludePorts is the list of ports to exclude from enumeration - ExcludeIps string // Ips or cidr to be excluded from the scan - ExcludeIpsFile string // File containing Ips or cidr to exclude from the scan - TopPorts string // Tops ports to scan - SourceIP string // SourceIP to use in TCP packets - Interface string // Interface to use for TCP packets - ConfigFile string // Config file contains a scan configuration - NmapCLI string // Nmap command (has priority over config file) - Threads int // Internal worker threads - EnableProgressBar bool // Enable progress bar - ScanAllIPS bool // Scan all the ips - ScanType string // Scan Type - Proxy string // Socks5 proxy - Resolvers string // Resolvers (comma separated or file) + HostsFile string // HostsFile is the file containing list of hosts to find port for + Output string // Output is the file to write found ports to. + Ports string // Ports is the ports to use for enumeration + PortsFile string // PortsFile is the file containing ports to use for enumeration + ExcludePorts string // ExcludePorts is the list of ports to exclude from enumeration + ExcludeIps string // Ips or cidr to be excluded from the scan + ExcludeIpsFile string // File containing Ips or cidr to exclude from the scan + TopPorts string // Tops ports to scan + SourceIP string // SourceIP to use in TCP packets + Interface string // Interface to use for TCP packets + ConfigFile string // Config file contains a scan configuration + NmapCLI string // Nmap command (has priority over config file) + Threads int // Internal worker threads + EnableProgressBar bool // Enable progress bar + ScanAllIPS bool // Scan all the ips + ScanType string // Scan Type + Proxy string // Socks5 proxy + Resolvers string // Resolvers (comma separated or file) baseResolvers []string OnResult OnResultCallback // OnResult callback } @@ -62,57 +62,57 @@ func ParseOptions() *Options { flagSet.SetDescription(`Naabu is a port scanning tool written in Go that allows you to enumerate open ports for hosts in a fast and reliable manner.`) createGroup(flagSet, "input", "Input", - flagSet.NormalizedStringSliceVarP(&options.Host, "host", "",[]string{}, "Single Host or comma-separated list of hosts to scan ports for"), - flagSet.StringVarP(&options.HostsFile, "l", "list", "", "File containing list of hosts to scan ports"), - flagSet.StringVarP(&options.ExcludeIps, "eh", "exclude-hosts", "", "Specifies a comma-separated list of targets to be excluded from the scan (ip, cidr)"), - flagSet.StringVarP(&options.ExcludeIpsFile, "ef", "exclude-file", "", "Specifies a newline-delimited file with targets to be excluded from the scan (ip, cidr)"), + flagSet.NormalizedStringSliceVarP(&options.Host, "host", "", []string{}, "hosts to scan ports for (comma-separated)"), + flagSet.StringVarP(&options.HostsFile, "l", "list", "", "list of hosts to scan ports (file)"), + flagSet.StringVarP(&options.ExcludeIps, "eh", "exclude-hosts", "", "hosts to exclude from the scan (comma-separated)"), + flagSet.StringVarP(&options.ExcludeIpsFile, "ef", "exclude-file", "", "list of hosts to exclude from scan (file)"), ) createGroup(flagSet, "port", "Port", - flagSet.StringVarP(&options.Ports, "p", "port", "", "Ports to scan (80, 80,443, 100-200"), - flagSet.StringVarP(&options.TopPorts, "tp", "top-ports", "", "Top Ports to scan (default top 100)"), - flagSet.StringVarP(&options.ExcludePorts, "ep", "exclude-ports", "", "Ports to exclude from scan"), - flagSet.StringVarP(&options.PortsFile, "pf", "ports-file", "", "File containing ports to scan for"), - flagSet.BoolVarP(&options.ExcludeCDN, "ec", "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)"), + flagSet.StringVarP(&options.Ports, "p", "port", "", "ports to scan (80,443, 100-200"), + flagSet.StringVarP(&options.TopPorts, "tp", "top-ports", "", "top ports to scan (default 100)"), + flagSet.StringVarP(&options.ExcludePorts, "ep", "exclude-ports", "", "ports to exclude from scan (comma-separated)"), + flagSet.StringVarP(&options.PortsFile, "pf", "ports-file", "", "list of ports to exclude from scan (file)"), + flagSet.BoolVarP(&options.ExcludeCDN, "ec", "exclude-cdn", false, "skip full port scans for CDN's (only checks for 80,443)"), ) createGroup(flagSet, "rate-limit", "Rate-limit", - flagSet.IntVar(&options.Threads, "c", 25, "General internal worker threads"), - flagSet.IntVar(&options.Rate, "rate", DefaultRateSynScan, "Rate of port scan probe request"), + flagSet.IntVar(&options.Threads, "c", 25, "general internal worker threads"), + flagSet.IntVar(&options.Rate, "rate", DefaultRateSynScan, "packets to send per second"), ) createGroup(flagSet, "output", "Output", - flagSet.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)"), - flagSet.BoolVar(&options.JSON, "json", false, "Write output in JSON lines Format"), + flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output to (optional)"), + flagSet.BoolVar(&options.JSON, "json", false, "write output in JSON lines format"), ) createGroup(flagSet, "config", "Configuration", - flagSet.BoolVar(&options.ScanAllIPS, "scan-all-ips", false, "Scan all the ips"), - flagSet.StringVarP(&options.ScanType, "s", "scan-type", SynScan, "Port scan type (SYN/CONNECT)"), - flagSet.StringVar(&options.SourceIP, "source-ip", "", "Source Ip"), - flagSet.BoolVarP(&options.InterfacesList, "il", "interface-list", false, "List available interfaces and public ip"), - flagSet.StringVarP(&options.Interface, "i", "interface", "", "Network Interface to use for port scan"), - flagSet.BoolVar(&options.Nmap, "nmap", false, "Invoke nmap scan on targets (nmap must be installed) - Deprecated"), + flagSet.BoolVarP(&options.ScanAllIPS, "sa", "scan-all-ips", false, "scan all the IP's associated with DNS record"), + flagSet.StringVarP(&options.ScanType, "s", "scan-type", SynScan, "type of port scan (SYN/CONNECT)"), + flagSet.StringVar(&options.SourceIP, "source-ip", "", "source ip"), + flagSet.BoolVarP(&options.InterfacesList, "il", "interface-list", false, "list available interfaces and public ip"), + flagSet.StringVarP(&options.Interface, "i", "interface", "", "network Interface to use for port scan"), + flagSet.BoolVar(&options.Nmap, "nmap", false, "invoke nmap scan on targets (nmap must be installed) - Deprecated"), flagSet.StringVar(&options.NmapCLI, "nmap-cli", "", "nmap command to run on found results (example: -nmap-cli 'nmap -sV')"), - flagSet.StringVar(&options.Resolvers, "r", "", "Custom resolvers to use to resolve DNS names (comma separated or from file)"), - flagSet.StringVar(&options.Proxy, "proxy", "", "Socks5 proxy"), + flagSet.StringVar(&options.Resolvers, "r", "", "list of custom resolver dns resolution (comma separated or from file)"), + flagSet.StringVar(&options.Proxy, "proxy", "", "socks5 proxy"), ) createGroup(flagSet, "optimization", "Optimization", - flagSet.IntVar(&options.Retries, "retries", DefaultRetriesSynScan, "Number of retries for the port scan probe"), - flagSet.IntVar(&options.Timeout, "timeout", DefaultPortTimeoutSynScan, "Millisecond to wait before timing out"), - flagSet.IntVar(&options.WarmUpTime, "warm-up-time", 2, "Time in seconds between scan phases"), - flagSet.BoolVar(&options.Ping, "ping", false, "Use ping probes for verification of host"), - flagSet.BoolVar(&options.Verify, "verify", false, "Validate the ports again with TCP verification"), + flagSet.IntVar(&options.Retries, "retries", DefaultRetriesSynScan, "number of retries for the port scan"), + flagSet.IntVar(&options.Timeout, "timeout", DefaultPortTimeoutSynScan, "millisecond to wait before timing out"), + flagSet.IntVar(&options.WarmUpTime, "warm-up-time", 2, "time in seconds between scan phases"), + flagSet.BoolVar(&options.Ping, "ping", false, "ping probes for verification of host"), + flagSet.BoolVar(&options.Verify, "verify", false, "validate the ports again with TCP verification"), ) createGroup(flagSet, "debug", "Debug", - flagSet.BoolVar(&options.Debug, "debug", false, "Enable debugging information"), - flagSet.BoolVar(&options.Verbose, "v", false, "Show Verbose output"), - flagSet.BoolVarP(&options.NoColor, "nc", "no-color", false, "Don't Use colors in output"), - flagSet.BoolVar(&options.Silent, "silent", false, "Show found ports only in output"), - flagSet.BoolVar(&options.Version, "version", false, "Show version of naabu"), - flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan"), + flagSet.BoolVar(&options.Debug, "debug", false, "display debugging information"), + flagSet.BoolVarP(&options.Verbose, "v", "verbose", false, "display verbose output"), + flagSet.BoolVarP(&options.NoColor, "nc", "no-color", false, "disable colors in CLI output"), + flagSet.BoolVar(&options.Silent, "silent", false, "display only results in output"), + flagSet.BoolVar(&options.Version, "version", false, "display version of naabu"), + flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display stats of the running scan"), ) _ = flagSet.Parse() From 7852ae5bceaeb2158a94be45ab485a852ab78bce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 13:04:22 +0000 Subject: [PATCH 37/67] chore(deps): bump golang from 1.17.5-alpine to 1.17.6-alpine Bumps golang from 1.17.5-alpine to 1.17.6-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7882ca49..301ad8e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.5-alpine AS builder +FROM golang:1.17.6-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From 96ca18a1cb241ec4bbaa0d71a996d9f945d100bf Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 17 Jan 2022 17:04:07 +0100 Subject: [PATCH 38/67] Adding support for linux capabilities check --- v2/go.mod | 3 ++- v2/go.sum | 2 ++ v2/pkg/privileges/privileges.go | 7 +++++++ v2/pkg/privileges/privileges_unix.go | 21 +++++++++++++++++++++ v2/pkg/privileges/privileges_win.go | 8 ++++++++ v2/pkg/runner/banners.go | 3 ++- v2/pkg/runner/runner.go | 8 ++++---- v2/pkg/runner/targets.go | 5 +++-- v2/pkg/runner/util.go | 5 ----- v2/pkg/runner/validate.go | 7 ++++--- v2/pkg/scan/option.go | 1 - v2/pkg/scan/scan.go | 4 ++-- v2/pkg/scan/scan_unix.go | 2 +- 13 files changed, 56 insertions(+), 20 deletions(-) create mode 100644 v2/pkg/privileges/privileges.go create mode 100644 v2/pkg/privileges/privileges_unix.go create mode 100644 v2/pkg/privileges/privileges_win.go diff --git a/v2/go.mod b/v2/go.mod index 3b942e36..2acfafc0 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -26,8 +26,10 @@ require ( ) require ( + github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 + github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 ) require ( @@ -36,7 +38,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/karrick/godirwalk v1.16.1 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.43 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 118aaede..7d98f728 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -131,6 +131,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= diff --git a/v2/pkg/privileges/privileges.go b/v2/pkg/privileges/privileges.go new file mode 100644 index 00000000..6fc32cdb --- /dev/null +++ b/v2/pkg/privileges/privileges.go @@ -0,0 +1,7 @@ +package privileges + +var IsPrivileged bool + +func init() { + IsPrivileged = isPrivileged() +} diff --git a/v2/pkg/privileges/privileges_unix.go b/v2/pkg/privileges/privileges_unix.go new file mode 100644 index 00000000..59e9fe31 --- /dev/null +++ b/v2/pkg/privileges/privileges_unix.go @@ -0,0 +1,21 @@ +//go:build linux || darwin + +package privileges + +import ( + "os" + + "github.com/syndtr/gocapability/capability" +) + +// isPrivileged checks if the current process has the CAP_NET_RAW capability or is root +func isPrivileged() bool { + caps, err := capability.NewPid2(0) + if err == nil { + if err := caps.Load(); err == nil { + return caps.Get(capability.EFFECTIVE, capability.CAP_NET_RAW) + } + } + + return os.Geteuid() == 0 +} diff --git a/v2/pkg/privileges/privileges_win.go b/v2/pkg/privileges/privileges_win.go new file mode 100644 index 00000000..f9efccff --- /dev/null +++ b/v2/pkg/privileges/privileges_win.go @@ -0,0 +1,8 @@ +//go:build windows + +package privileges + +// IsPrivileged on windows doesn't matter as we are using connect scan +func isPrivileged() bool { + return false +} diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index d7a6f346..13972631 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/naabu/v2/pkg/privileges" "github.com/projectdiscovery/naabu/v2/pkg/scan" ) @@ -31,7 +32,7 @@ func showBanner() { func showNetworkCapabilities(options *Options) { accessLevel := "non root" scanType := "CONNECT" - if isRoot() && options.ScanType == SynScan { + if privileges.IsPrivileged && options.ScanType == SynScan { accessLevel = "root" scanType = "SYN" } diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 84a13779..9a0df812 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ipranger" "github.com/projectdiscovery/mapcidr" + "github.com/projectdiscovery/naabu/v2/pkg/privileges" "github.com/projectdiscovery/naabu/v2/pkg/scan" "github.com/remeh/sizedwaitgroup" "go.uber.org/ratelimit" @@ -54,7 +55,6 @@ func NewRunner(options *Options) (*Runner, error) { Retries: options.Retries, Rate: options.Rate, Debug: options.Debug, - Root: isRoot(), ExcludeCdn: options.ExcludeCDN, ExcludedIps: excludedIps, Proxy: options.Proxy, @@ -100,7 +100,7 @@ func NewRunner(options *Options) (*Runner, error) { func (r *Runner) RunEnumeration() error { defer r.Close() - if isRoot() && r.options.ScanType == SynScan { + if privileges.IsPrivileged && r.options.ScanType == SynScan { // Set values if those were specified via cli if err := r.SetSourceIPAndInterface(); err != nil { // Otherwise try to obtain them automatically @@ -153,7 +153,7 @@ func (r *Runner) RunEnumeration() error { } } - osSupported := isOSSupported() + shouldUseRawPackets := isOSSupported() && privileges.IsPrivileged && r.options.ScanType == SynScan // Retries are performed regardless of the previous scan results due to network unreliability for currentRetry := 0; currentRetry < r.options.Retries; currentRetry++ { // Use current time as seed @@ -167,7 +167,7 @@ func (r *Runner) RunEnumeration() error { r.limiter.Take() // connect scan - if osSupported && isRoot() && r.options.ScanType == SynScan { + if shouldUseRawPackets { r.RawSocketEnumeration(ip, port) } else { r.wgscan.Add() diff --git a/v2/pkg/runner/targets.go b/v2/pkg/runner/targets.go index 1f3a76ad..37bdd138 100644 --- a/v2/pkg/runner/targets.go +++ b/v2/pkg/runner/targets.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ipranger" + "github.com/projectdiscovery/naabu/v2/pkg/privileges" "github.com/projectdiscovery/naabu/v2/pkg/scan" "github.com/remeh/sizedwaitgroup" ) @@ -44,7 +45,7 @@ func (r *Runner) mergeToFile() (string, error) { // target defined via CLI argument if len(r.options.Host) > 0 { - for _,v :=range r.options.Host { + for _, v := range r.options.Host { fmt.Fprintf(tempInput, "%s\n", v) } } @@ -151,7 +152,7 @@ func (r *Runner) resolveFQDN(target string) ([]string, error) { } // If the user has specified ping probes, perform ping on addresses - if isRoot() && r.options.Ping && len(initialHosts) > 1 { + if privileges.IsPrivileged && r.options.Ping && len(initialHosts) > 1 { // Scan the hosts found for ping probes pingResults, err := scan.PingHosts(initialHosts) if err != nil { diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go index 0573cfff..7c8967ff 100644 --- a/v2/pkg/runner/util.go +++ b/v2/pkg/runner/util.go @@ -2,17 +2,12 @@ package runner import ( "fmt" - "os" "runtime" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/iputil" ) -func isRoot() bool { - return os.Geteuid() == 0 -} - func (r *Runner) host2ips(target string) (targetIPs []string, err error) { // If the host is a Domain, then perform resolution and discover all IP // addresses for a given host. Else use that host for port scanning diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index 2f1d12c1..4a3736b7 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/naabu/v2/pkg/privileges" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -35,17 +36,17 @@ func (options *Options) validateOptions() error { if options.Timeout == 0 { return errors.Wrap(errZeroValue, "timeout") - } else if !isRoot() && options.Timeout == DefaultPortTimeoutSynScan { + } else if !privileges.IsPrivileged && options.Timeout == DefaultPortTimeoutSynScan { options.Timeout = DefaultPortTimeoutConnectScan } if options.Rate == 0 { return errors.Wrap(errZeroValue, "rate") - } else if !isRoot() && options.Rate == DefaultRateSynScan { + } else if !privileges.IsPrivileged && options.Rate == DefaultRateSynScan { options.Rate = DefaultRateConnectScan } - if !isRoot() && options.Retries == DefaultRetriesSynScan { + if !privileges.IsPrivileged && options.Retries == DefaultRetriesSynScan { options.Retries = DefaultRetriesConnectScan } diff --git a/v2/pkg/scan/option.go b/v2/pkg/scan/option.go index cc1ddc0c..ef92be31 100644 --- a/v2/pkg/scan/option.go +++ b/v2/pkg/scan/option.go @@ -10,7 +10,6 @@ type Options struct { Retries int Rate int Debug bool - Root bool ExcludeCdn bool ExcludedIps []string Proxy string diff --git a/v2/pkg/scan/scan.go b/v2/pkg/scan/scan.go index bd84ebcf..e68a6c8f 100644 --- a/v2/pkg/scan/scan.go +++ b/v2/pkg/scan/scan.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ipranger" + "github.com/projectdiscovery/naabu/v2/pkg/privileges" "github.com/projectdiscovery/naabu/v2/pkg/result" "github.com/projectdiscovery/networkpolicy" "golang.org/x/net/icmp" @@ -129,7 +130,7 @@ func NewScanner(options *Options) (*Scanner, error) { IPRanger: iprang, } - if options.Root && newScannerCallback != nil { + if privileges.IsPrivileged && newScannerCallback != nil { if err := newScannerCallback(scanner); err != nil { return nil, err } @@ -532,7 +533,6 @@ func (s *Scanner) SetupHandlers() error { if s.NetworkInterface != nil { return s.SetupHandler(s.NetworkInterface.Name) } - itfs, err := net.Interfaces() if err != nil { return err diff --git a/v2/pkg/scan/scan_unix.go b/v2/pkg/scan/scan_unix.go index 4b0c326b..8b3419da 100644 --- a/v2/pkg/scan/scan_unix.go +++ b/v2/pkg/scan/scan_unix.go @@ -1,4 +1,4 @@ -// +build linux darwin +//go:build linux || darwin package scan From b11a68182be42b010816a5c35b5faf544d007f81 Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Mon, 17 Jan 2022 17:58:17 +0100 Subject: [PATCH 39/67] adding runtime load of CAP_NET_RAW --- v2/pkg/privileges/privileges_unix.go | 21 +++++++++++++++------ v2/pkg/runner/banners.go | 4 ++++ v2/pkg/runner/util.go | 10 +++++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/v2/pkg/privileges/privileges_unix.go b/v2/pkg/privileges/privileges_unix.go index 59e9fe31..46078ca2 100644 --- a/v2/pkg/privileges/privileges_unix.go +++ b/v2/pkg/privileges/privileges_unix.go @@ -4,18 +4,27 @@ package privileges import ( "os" + "runtime" - "github.com/syndtr/gocapability/capability" + "golang.org/x/sys/unix" ) // isPrivileged checks if the current process has the CAP_NET_RAW capability or is root func isPrivileged() bool { - caps, err := capability.NewPid2(0) - if err == nil { - if err := caps.Load(); err == nil { - return caps.Get(capability.EFFECTIVE, capability.CAP_NET_RAW) - } + header := unix.CapUserHeader{ + Version: unix.LINUX_CAPABILITY_VERSION_3, + Pid: int32(os.Getpid()), } + data := unix.CapUserData{} + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + if err := unix.Capget(&header, &data); err == nil { + data.Inheritable = (1 << unix.CAP_NET_RAW) + if err := unix.Capset(&header, &data); err == nil { + return true + } + } return os.Geteuid() == 0 } diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 13972631..7feeaf0f 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -34,6 +34,10 @@ func showNetworkCapabilities(options *Options) { scanType := "CONNECT" if privileges.IsPrivileged && options.ScanType == SynScan { accessLevel = "root" + if isLinux() { + accessLevel = "CAP_NET_RAW" + } + scanType = "SYN" } gologger.Info().Msgf("Running %s scan with %s privileges\n", scanType, accessLevel) diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go index 7c8967ff..37a834fb 100644 --- a/v2/pkg/runner/util.go +++ b/v2/pkg/runner/util.go @@ -36,7 +36,15 @@ func (r *Runner) host2ips(target string) (targetIPs []string, err error) { } func isOSSupported() bool { - return runtime.GOOS == "linux" || runtime.GOOS == "darwin" + return isLinux() || isOSX() +} + +func isOSX() bool { + return runtime.GOOS == "darwin" +} + +func isLinux() bool { + return runtime.GOOS == "linux" } func mapKeysToSliceInt(m map[int]struct{}) (s []int) { From c554ef1c58a42566fb385fab95ecc351298ab924 Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Mon, 17 Jan 2022 17:59:11 +0100 Subject: [PATCH 40/67] updating go modules --- v2/go.mod | 3 +-- v2/go.sum | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 2acfafc0..d873e68c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,7 +21,7 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 go.uber.org/ratelimit v0.2.0 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect + golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) @@ -29,7 +29,6 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 ) require ( diff --git a/v2/go.sum b/v2/go.sum index 7d98f728..118aaede 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -131,8 +131,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= From 3dde5df1b2cbcec1ae060411a3caf1f1e7d5ad17 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 25 Jan 2022 18:09:36 +0100 Subject: [PATCH 41/67] fixing compilation error on darwin --- v2/pkg/privileges/privileges_darwin.go | 12 ++++++++++++ .../{privileges_unix.go => privileges_linux.go} | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/privileges/privileges_darwin.go rename v2/pkg/privileges/{privileges_unix.go => privileges_linux.go} (95%) diff --git a/v2/pkg/privileges/privileges_darwin.go b/v2/pkg/privileges/privileges_darwin.go new file mode 100644 index 00000000..1a10639f --- /dev/null +++ b/v2/pkg/privileges/privileges_darwin.go @@ -0,0 +1,12 @@ +//go:build darwin + +package privileges + +import ( + "os" +) + +// isPrivileged checks if the current process has the CAP_NET_RAW capability or is root +func isPrivileged() bool { + return os.Geteuid() == 0 +} diff --git a/v2/pkg/privileges/privileges_unix.go b/v2/pkg/privileges/privileges_linux.go similarity index 95% rename from v2/pkg/privileges/privileges_unix.go rename to v2/pkg/privileges/privileges_linux.go index 46078ca2..c83c9e9c 100644 --- a/v2/pkg/privileges/privileges_unix.go +++ b/v2/pkg/privileges/privileges_linux.go @@ -1,4 +1,4 @@ -//go:build linux || darwin +//go:build linux || unix package privileges From b2992521f67f565631388718e991023c06134783 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 31 Jan 2022 08:20:52 -0600 Subject: [PATCH 42/67] timestamp for json output --- v2/pkg/runner/output.go | 10 ++++++---- v2/pkg/runner/runner.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/v2/pkg/runner/output.go b/v2/pkg/runner/output.go index 12ae684e..630ce8ec 100644 --- a/v2/pkg/runner/output.go +++ b/v2/pkg/runner/output.go @@ -6,13 +6,15 @@ import ( "io" "strconv" "strings" + "time" ) // JSONResult contains the result for a host in JSON format type JSONResult struct { - Host string `json:"host,omitempty"` - IP string `json:"ip,omitempty"` - Port int `json:"port"` + Host string `json:"host,omitempty"` + IP string `json:"ip,omitempty"` + Port int `json:"port"` + TimeStamp time.Time `json:"timestamp"` } // WriteHostOutput writes the output list of host ports to an io.Writer @@ -40,7 +42,7 @@ func WriteHostOutput(host string, ports map[int]struct{}, writer io.Writer) erro func WriteJSONOutput(host, ip string, ports map[int]struct{}, writer io.Writer) error { encoder := json.NewEncoder(writer) - data := JSONResult{} + data := JSONResult{TimeStamp: time.Now().UTC()} if host != ip { data.Host = host } diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 9a0df812..477e77f4 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -355,7 +355,7 @@ func (r *Runner) handleOutput() { // console output if r.options.JSON { - data := JSONResult{IP: hostIP} + data := JSONResult{IP: hostIP, TimeStamp: time.Now().UTC()} if host != hostIP { data.Host = host } From 4d6f1d42fce8957a5a8589cd13f4dcc779ce3fb4 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 1 Feb 2022 06:50:26 +0100 Subject: [PATCH 43/67] updating go mods --- v2/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index d873e68c..415e8a6b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,7 +3,6 @@ module github.com/projectdiscovery/naabu/v2 go 1.17 require ( - github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e @@ -22,7 +21,6 @@ require ( go.uber.org/ratelimit v0.2.0 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) require ( @@ -35,6 +33,7 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/miekg/dns v1.1.43 // indirect @@ -50,4 +49,5 @@ require ( go.uber.org/atomic v1.9.0 // indirect golang.org/x/text v0.3.7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) From a24a75ef41bf1d273bb5a9b36cc40a256ada691f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 1 Feb 2022 07:02:45 +0100 Subject: [PATCH 44/67] adding dependencies setup to lint gh action --- .github/workflows/lint-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 9e39da2c..283820d3 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -12,6 +12,9 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: Install Dependences + run: sudo apt install libpcap-dev + - name: Run golangci-lint uses: golangci/golangci-lint-action@v2 with: From 90f3847945dd3f9296e81683c4a13be282adb817 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 2 Feb 2022 06:48:02 +0100 Subject: [PATCH 45/67] adding customizable stats interval --- v2/pkg/runner/default.go | 5 +++-- v2/pkg/runner/options.go | 2 ++ v2/pkg/runner/runner.go | 6 +----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/v2/pkg/runner/default.go b/v2/pkg/runner/default.go index 59438851..e13e068d 100644 --- a/v2/pkg/runner/default.go +++ b/v2/pkg/runner/default.go @@ -12,6 +12,7 @@ const ( ExternalTargetForTune = "8.8.8.8" - SynScan = "s" - ConnectScan = "c" + SynScan = "s" + ConnectScan = "c" + DefautStatsInterval = 5 ) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 34dcce98..fc8c2d4b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -49,6 +49,7 @@ type Options struct { Resolvers string // Resolvers (comma separated or file) baseResolvers []string OnResult OnResultCallback // OnResult callback + StatsInterval int // StatsInterval is the number of seconds to display stats after } // OnResultCallback (hostname, ip, ports) @@ -113,6 +114,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Silent, "silent", false, "display only results in output"), flagSet.BoolVar(&options.Version, "version", false, "display version of naabu"), flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display stats of the running scan"), + flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", DefautStatsInterval, "number of seconds to wait between showing a statistics update"), ) _ = flagSet.Parse() diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 9a0df812..9077c833 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -22,10 +22,6 @@ import ( "go.uber.org/ratelimit" ) -const ( - tickduration = 5 -) - // Runner is an instance of the port enumeration // client used to orchestrate the whole process. type Runner struct { @@ -148,7 +144,7 @@ func (r *Runner) RunEnumeration() error { r.stats.AddCounter("packets", uint64(0)) r.stats.AddCounter("errors", uint64(0)) r.stats.AddCounter("total", Range*uint64(r.options.Retries)) - if err := r.stats.Start(makePrintCallback(), tickduration*time.Second); err != nil { + if err := r.stats.Start(makePrintCallback(), time.Duration(r.options.StatsInterval)*time.Second); err != nil { gologger.Warning().Msgf("Couldn't start statistics: %s\n", err) } } From 9f17295ec966149dea9acdca689ebec2535ffeb3 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 9 Feb 2022 07:24:02 +0100 Subject: [PATCH 46/67] Updating GH workflow + sonarcloud (#268) * update dep bot * updating gh actions * adding pcap * fixing paths --- .github/dependabot.yml | 12 ++++++-- .github/workflows/build-test.yml | 23 ++++++++++---- .github/workflows/codeql-analysis.yml | 1 + .github/workflows/functional-test.yml | 22 +++++++------- .github/workflows/sonarcloud.yml | 43 +++++++++++++++++++++++++++ sonar-project.properties | 16 ++++++++++ 6 files changed, 98 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/sonarcloud.yml create mode 100644 sonar-project.properties diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 69d95437..89561d73 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,16 +15,20 @@ updates: commit-message: prefix: "chore" include: "scope" + labels: + - "Type: Maintenance" # Maintain dependencies for go modules - package-ecosystem: "gomod" - directory: "/" + directory: "v2/" schedule: - interval: "weekly" + interval: "daily" target-branch: "dev" commit-message: prefix: "chore" include: "scope" + labels: + - "Type: Maintenance" # Maintain dependencies for docker - package-ecosystem: "docker" @@ -34,4 +38,6 @@ updates: target-branch: "dev" commit-message: prefix: "chore" - include: "scope" \ No newline at end of file + include: "scope" + labels: + - "Type: Maintenance" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6b63bc66..6dbfcc87 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -4,11 +4,13 @@ on: pull_request: workflow_dispatch: - jobs: build: - name: Build Test - runs-on: ubuntu-latest + name: Test Builds + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] # Todo: windows-latest, macOS-latest steps: - name: Set up Go uses: actions/setup-go@v2 @@ -24,10 +26,21 @@ jobs: - name: Install nmap run: sudo apt install nmap + - name: Build + run: go build . + working-directory: v2/cmd/naabu/ + - name: Test run: go test ./... working-directory: v2/ - - name: Build - run: go build . + # Todo + # - name: Integration Tests + # env: + # GH_ACTION: true + # run: bash run.sh + # working-directory: integration_tests/ + + - name: Race Condition Tests + run: go build -race . working-directory: v2/cmd/naabu/ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9ea93430..fcdd108b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,6 +2,7 @@ name: 🚨 CodeQL Analysis on: workflow_dispatch: + push: pull_request: branches: - dev diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 583323ce..8d1bdcbe 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -4,29 +4,29 @@ on: pull_request: workflow_dispatch: - jobs: functional: name: Functional Test - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] # Todo: windows-latest, macOS-latest + steps: - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.17 - - name: Install libpcap-dev - run: sudo apt install libpcap-dev - - - name: Install nmap - run: sudo apt install nmap - - - name: Install simplehttpserver - run: go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest - - name: Check out code uses: actions/checkout@v2 + - name: Install required packages (libpcap-dev, nmap, simplehttpserver) + run: | + sudo apt install libpcap-dev + sudo apt install nmap + go install github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@latest + - name: Run the simplehttpserver on 127.0.0.1:8000 run: | export PATH=$PATH:$(go env GOPATH)/bin diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml new file mode 100644 index 00000000..6c046c39 --- /dev/null +++ b/.github/workflows/sonarcloud.yml @@ -0,0 +1,43 @@ +name: 👮🏼‍♂️ Sonarcloud +on: + push: + branches: + - master + - dev + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + sonarcloud: + name: SonarCloud + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + + - name: "Set up Go" + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Install Dependences + run: sudo apt install libpcap-dev + + - name: Run unit Tests + working-directory: v2/ + run: | + go test -coverprofile=cov.out ./... + + - name: Run Gosec Security Scanner + working-directory: v2/ + run: | + go install github.com/securego/gosec/cmd/gosec@latest + gosec -no-fail -fmt=sonarqube -out report.json ./... + + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 00000000..6f301c11 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,16 @@ +sonar.projectKey=projectdiscovery_naabu +sonar.organization=projectdiscovery + +# This is the name and version displayed in the SonarCloud UI. +#sonar.projectName=dnsx +#sonar.projectVersion=1.0 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +sonar.sources=v2/ +sonar.tests=v2/ +sonar.test.inclusions=**/*_test.go +sonar.go.coverage.reportPaths=v2/cov.out +sonar.externalIssuesReportPaths=v2/report.json + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 \ No newline at end of file From 188a01b8b72a1d1e890769a8b8a8fb8f473d5d0d Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 10 Feb 2022 06:58:51 -0600 Subject: [PATCH 47/67] Csv support (#269) * csv support * merge conflits and timestamp add * lint error fix * refactoring result struct Co-authored-by: mzack --- v2/pkg/runner/options.go | 4 ++- v2/pkg/runner/output.go | 70 ++++++++++++++++++++++++++++++++++----- v2/pkg/runner/runner.go | 65 ++++++++++++++++++++++++++++++------ v2/pkg/runner/validate.go | 11 ++++-- 4 files changed, 127 insertions(+), 23 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index fc8c2d4b..cd2f6d9d 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -49,7 +49,8 @@ type Options struct { Resolvers string // Resolvers (comma separated or file) baseResolvers []string OnResult OnResultCallback // OnResult callback - StatsInterval int // StatsInterval is the number of seconds to display stats after + CSV bool + StatsInterval int // StatsInterval is the number of seconds to display stats after } // OnResultCallback (hostname, ip, ports) @@ -85,6 +86,7 @@ func ParseOptions() *Options { createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output to (optional)"), flagSet.BoolVar(&options.JSON, "json", false, "write output in JSON lines format"), + flagSet.BoolVar(&options.CSV, "csv", false, "write output in csv format"), ) createGroup(flagSet, "config", "Configuration", diff --git a/v2/pkg/runner/output.go b/v2/pkg/runner/output.go index 630ce8ec..25a65ebd 100644 --- a/v2/pkg/runner/output.go +++ b/v2/pkg/runner/output.go @@ -2,19 +2,54 @@ package runner import ( "bufio" + "encoding/csv" "encoding/json" + "fmt" "io" + "reflect" "strconv" "strings" "time" + + "github.com/pkg/errors" ) -// JSONResult contains the result for a host in JSON format -type JSONResult struct { - Host string `json:"host,omitempty"` - IP string `json:"ip,omitempty"` - Port int `json:"port"` - TimeStamp time.Time `json:"timestamp"` +// Result contains the result for a host +type Result struct { + Host string `json:"host,omitempty" csv:"host"` + IP string `json:"ip,omitempty" csv:"ip"` + Port int `json:"port" csv:"port"` + TimeStamp time.Time `json:"timestamp" csv:"timestamp"` +} + +func (r *Result) JSON() ([]byte, error) { + return json.Marshal(r) +} + +var NumberOfCsvFieldsErr = errors.New("exported fields don't match csv tags") + +func (r *Result) CSVHeaders() ([]string, error) { + ty := reflect.TypeOf(*r) + var headers []string + for i := 0; i < ty.NumField(); i++ { + headers = append(headers, ty.Field(i).Tag.Get("csv")) + } + if len(headers) != ty.NumField() { + return nil, NumberOfCsvFieldsErr + } + return headers, nil +} + +func (r *Result) CSVFields() ([]string, error) { + var fields []string + vl := reflect.ValueOf(*r) + for i := 0; i < vl.NumField(); i++ { + fields = append(fields, fmt.Sprint(vl.Field(i).Interface())) + } + if len(fields) != vl.NumField() { + return nil, NumberOfCsvFieldsErr + } + return fields, nil } // WriteHostOutput writes the output list of host ports to an io.Writer @@ -42,7 +77,7 @@ func WriteHostOutput(host string, ports map[int]struct{}, writer io.Writer) erro func WriteJSONOutput(host, ip string, ports map[int]struct{}, writer io.Writer) error { encoder := json.NewEncoder(writer) - data := JSONResult{TimeStamp: time.Now().UTC()} + data := Result{TimeStamp: time.Now().UTC()} if host != ip { data.Host = host } @@ -50,7 +85,6 @@ func WriteJSONOutput(host, ip string, ports map[int]struct{}, writer io.Writer) for port := range ports { data.Port = port - err := encoder.Encode(&data) if err != nil { return err @@ -58,3 +92,23 @@ func WriteJSONOutput(host, ip string, ports map[int]struct{}, writer io.Writer) } return nil } + +// WriteCsvOutput writes the output list of subdomain in csv format to an io.Writer +func WriteCsvOutput(host, ip string, ports map[int]struct{}, header bool, writer io.Writer) error { + encoder := csv.NewWriter(writer) + data := &Result{TimeStamp: time.Now().UTC()} + if header { + writeCSVHeaders(data, encoder) + } + if host != ip { + data.Host = host + } + data.IP = ip + + for port := range ports { + data.Port = port + writeCSVRow(data, encoder) + } + encoder.Flush() + return nil +} diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index d364f5ec..44c64525 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -1,7 +1,8 @@ package runner import ( - "encoding/json" + "bytes" + "encoding/csv" "fmt" "net" "os" @@ -10,6 +11,7 @@ import ( "sync" "time" + "github.com/pkg/errors" "github.com/projectdiscovery/blackrock" "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/dnsx/libs/dnsx" @@ -336,43 +338,59 @@ func (r *Runner) handleOutput() { } defer file.Close() } - + csvFileHeaderEnabled := true for hostIP, ports := range r.scanner.ScanResults.IPPorts { + csvHeaderEnabled := true dt, err := r.scanner.IPRanger.GetHostsByIP(hostIP) if err != nil { continue } - + buffer := bytes.Buffer{} + writer := csv.NewWriter(&buffer) for _, host := range dt { if host == "ip" { host = hostIP } gologger.Info().Msgf("Found %d ports on host %s (%s)\n", len(ports), host, hostIP) - // console output - if r.options.JSON { - data := JSONResult{IP: hostIP, TimeStamp: time.Now().UTC()} + if r.options.JSON || r.options.CSV { + data := &Result{IP: hostIP, TimeStamp: time.Now().UTC()} if host != hostIP { data.Host = host } for port := range ports { data.Port = port - b, marshallErr := json.Marshal(data) - if marshallErr != nil { - continue + if r.options.JSON { + b, marshallErr := data.JSON() + if marshallErr != nil { + continue + } + buffer.Write([]byte(fmt.Sprintf("%s\n", b))) + } else if r.options.CSV { + if csvHeaderEnabled { + writeCSVHeaders(data, writer) + csvHeaderEnabled = false + } + writeCSVRow(data, writer) } - gologger.Silent().Msgf("%s\n", string(b)) } + } + if r.options.JSON { + gologger.Silent().Msgf("%s", buffer.String()) + } else if r.options.CSV { + writer.Flush() + gologger.Silent().Msgf("%s", buffer.String()) } else { for port := range ports { gologger.Silent().Msgf("%s:%d\n", host, port) } } - // file output if file != nil { if r.options.JSON { err = WriteJSONOutput(host, hostIP, ports, file) + } else if r.options.CSV { + err = WriteCsvOutput(host, hostIP, ports, csvFileHeaderEnabled, file) } else { err = WriteHostOutput(host, ports, file) } @@ -385,6 +403,31 @@ func (r *Runner) handleOutput() { r.options.OnResult(host, hostIP, mapKeysToSliceInt(ports)) } } + csvFileHeaderEnabled = false + } +} +func writeCSVHeaders(data *Result, writer *csv.Writer) { + headers, err := data.CSVHeaders() + if err != nil { + gologger.Error().Msgf(err.Error()) + return + } + + if err := writer.Write(headers); err != nil { + errMsg := errors.Wrap(err, "Could not write headers") + gologger.Error().Msgf(errMsg.Error()) + } +} + +func writeCSVRow(data *Result, writer *csv.Writer) { + rowData, err := data.CSVFields() + if err != nil { + gologger.Error().Msgf(err.Error()) + return + } + if err := writer.Write(rowData); err != nil { + errMsg := errors.Wrap(err, "Could not write row") + gologger.Error().Msgf(errMsg.Error()) } } diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index 4a3736b7..085e4130 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -16,9 +16,10 @@ import ( ) var ( - errNoInputList = errors.New("no input list provided") - errOutputMode = errors.New("both verbose and silent mode specified") - errZeroValue = errors.New("cannot be zero") + errNoInputList = errors.New("no input list provided") + errOutputMode = errors.New("both verbose and silent mode specified") + errZeroValue = errors.New("cannot be zero") + errTwoOutputMode = errors.New("both json and csv mode specified") ) // validateOptions validates the configuration options passed @@ -34,6 +35,10 @@ func (options *Options) validateOptions() error { return errOutputMode } + if options.JSON && options.CSV { + return errTwoOutputMode + } + if options.Timeout == 0 { return errors.Wrap(errZeroValue, "timeout") } else if !privileges.IsPrivileged && options.Timeout == DefaultPortTimeoutSynScan { From 156a8cdb5d696ff97e527d9159cea517375a3cec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Feb 2022 11:45:34 +0530 Subject: [PATCH 48/67] chore(deps): bump golang from 1.17.6-alpine to 1.17.7-alpine (#271) Bumps golang from 1.17.6-alpine to 1.17.7-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 301ad8e6..18313fb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.6-alpine AS builder +FROM golang:1.17.7-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From e55ec0f8cca33069305746d71f7f4f951d82e7c2 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 23 Feb 2022 22:48:08 -0600 Subject: [PATCH 49/67] stop-resume support --- v2/cmd/naabu/main.go | 22 ++++++++++ v2/go.mod | 6 ++- v2/go.sum | 7 ++++ v2/pkg/runner/options.go | 16 ++++++- v2/pkg/runner/resume.go | 82 ++++++++++++++++++++++++++++++++++++ v2/pkg/runner/runner.go | 91 +++++++++++++++++++++++++++++++++------- 6 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 v2/pkg/runner/resume.go diff --git a/v2/cmd/naabu/main.go b/v2/cmd/naabu/main.go index 462edd2c..8b985aa2 100644 --- a/v2/cmd/naabu/main.go +++ b/v2/cmd/naabu/main.go @@ -4,6 +4,8 @@ import ( _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/naabu/v2/pkg/runner" + "os" + "os/signal" ) func main() { @@ -14,9 +16,29 @@ func main() { if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) } + // Setup graceful exits + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for range c { + naabuRunner.ShowScanResultOnExit() + gologger.Info().Msgf("CTRL+C pressed: Exiting\n") + if options.ResumeCfg.ShouldSaveResume() { + gologger.Info().Msgf("Creating resume file: %s\n", runner.DefaultResumeFilePath()) + err := options.ResumeCfg.SaveResumeConfig() + if err != nil { + gologger.Error().Msgf("Couldn't create resume file: %s\n", err) + } + } + naabuRunner.Close() + os.Exit(1) + } + }() err = naabuRunner.RunEnumeration() if err != nil { gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) } + // on successful execution remove the resume file in case it exists + options.ResumeCfg.CleanupResumeConfig() } diff --git a/v2/go.mod b/v2/go.mod index 415e8a6b..55bf305c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,7 @@ require ( github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/dnsx v1.0.7-0.20210927160546-05f957862698 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c + github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/ipranger v0.0.3-0.20210831161617-ac80efae0961 @@ -26,11 +26,13 @@ require ( require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 + github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/stretchr/testify v1.7.0 ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -41,6 +43,7 @@ require ( github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/hmap v0.0.1 // indirect + github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 // indirect @@ -48,6 +51,7 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect go.uber.org/atomic v1.9.0 // indirect golang.org/x/text v0.3.7 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 118aaede..5c1bc125 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,5 +1,7 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -88,6 +90,9 @@ github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0 github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc h1:dbDgsj26PW06L3zMo7AT08IbEqMd2u8QQ1BvlfMAY2w= +github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= +github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -110,6 +115,7 @@ github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vP github.com/projectdiscovery/networkpolicy v0.0.0-20210414200240-b3fa8abf324c/go.mod h1:KZIP5x7t+bH2dASgnYaqbiLI4z0gxXzekwUtarrQMdc= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d h1:tFPD9+3lq+XMHpQ/zMMp41RdXEl90h+KZli7UQt1LNY= github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d/go.mod h1:c5lCypH3Wv7PNRHFsTbmTWlOAV502VlqUmS9A/2wlNU= @@ -210,6 +216,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index cd2f6d9d..48fb6bcf 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -1,6 +1,7 @@ package runner import ( + "github.com/projectdiscovery/fileutil" "os" "github.com/projectdiscovery/goflags" @@ -51,6 +52,8 @@ type Options struct { OnResult OnResultCallback // OnResult callback CSV bool StatsInterval int // StatsInterval is the number of seconds to display stats after + Resume bool + ResumeCfg *ResumeCfg } // OnResultCallback (hostname, ip, ports) @@ -99,6 +102,7 @@ func ParseOptions() *Options { flagSet.StringVar(&options.NmapCLI, "nmap-cli", "", "nmap command to run on found results (example: -nmap-cli 'nmap -sV')"), flagSet.StringVar(&options.Resolvers, "r", "", "list of custom resolver dns resolution (comma separated or from file)"), flagSet.StringVar(&options.Proxy, "proxy", "", "socks5 proxy"), + flagSet.BoolVar(&options.Resume, "resume", false, "Resume"), ) createGroup(flagSet, "optimization", "Optimization", @@ -126,7 +130,12 @@ func ParseOptions() *Options { // Read the inputs and configure the logging options.configureOutput() - + options.ResumeCfg = NewResumeCfg() + if options.ShouldLoadResume() { + if err := options.ResumeCfg.ConfigureResume(); err != nil { + gologger.Fatal().Msgf("%s\n", err) + } + } // Show the user the banner showBanner() @@ -156,6 +165,11 @@ func ParseOptions() *Options { return options } +// ShouldLoadResume resume file +func (options *Options) ShouldLoadResume() bool { + return options.Resume && fileutil.FileExists(DefaultResumeFilePath()) +} + func hasStdin() bool { stat, err := os.Stdin.Stat() if err != nil { diff --git a/v2/pkg/runner/resume.go b/v2/pkg/runner/resume.go new file mode 100644 index 00000000..8afdf11a --- /dev/null +++ b/v2/pkg/runner/resume.go @@ -0,0 +1,82 @@ +package runner + +import ( + "encoding/json" + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "io/ioutil" + "os" + "path/filepath" + "sync" +) + +// Default resume file +const defaultResumeFileName = "resume.cfg" + +func DefaultResumeFilePath() string { + home, err := os.UserHomeDir() + if err != nil { + return defaultResumeFileName + } + return filepath.Join(home, ".config", "naabu", defaultResumeFileName) +} + +// ResumeCfg contains the scan progression +type ResumeCfg struct { + sync.RWMutex + ResumeFrom map[string]*ResumeInfo `json:"resumeFrom"` + Current map[string]*ResumeInfo `json:"-"` +} + +// InFlightResume +type InFlightResume struct { + Completed bool `json:"completed"` +} + +// ResumeInfo +type ResumeInfo struct { + sync.RWMutex + InFlight map[uint32]InFlightResume `json:"inFlight"` +} + +// NewResumeCfg creates a new scan progression structure +func NewResumeCfg() *ResumeCfg { + return &ResumeCfg{ + ResumeFrom: make(map[string]*ResumeInfo), + Current: make(map[string]*ResumeInfo), + } +} + +// SaveResumeConfig to file +func (resumeCfg *ResumeCfg) SaveResumeConfig() error { + newresumeCfg := NewResumeCfg() + newresumeCfg.ResumeFrom = resumeCfg.Current + data, _ := json.MarshalIndent(newresumeCfg, "", "\t") + return os.WriteFile(DefaultResumeFilePath(), data, os.ModePerm) +} + +// ConfigureResume read the resume config file +func (resumeCfg *ResumeCfg) ConfigureResume() error { + gologger.Info().Msg("Resuming from save checkpoint") + file, err := ioutil.ReadFile(DefaultResumeFilePath()) + if err != nil { + return err + } + err = json.Unmarshal([]byte(file), &resumeCfg) + if err != nil { + return err + } + return nil +} + +// ShouldSaveResume file +func (resumeCfg *ResumeCfg) ShouldSaveResume() bool { + return true +} + +// CleanupResumeConfig cleaning up the config file +func (resumeCfg *ResumeCfg) CleanupResumeConfig() { + if fileutil.FileExists(DefaultResumeFilePath()) { + os.Remove(DefaultResumeFilePath()) + } +} diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 44c64525..6e71578f 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -118,7 +118,6 @@ func (r *Runner) RunEnumeration() error { if err != nil { return err } - // Scan workers r.wgscan = sizedwaitgroup.New(r.options.Rate) r.limiter = ratelimit.New(r.options.Rate) @@ -162,18 +161,72 @@ func (r *Runner) RunEnumeration() error { portIndex := int(xxx % int64(portsCount)) ip := r.PickIP(targets, ipIndex) port := r.PickPort(portIndex) - r.limiter.Take() - // connect scan - if shouldUseRawPackets { - r.RawSocketEnumeration(ip, port) - } else { - r.wgscan.Add() - go r.handleHostPort(ip, port) + //resume cfg logic + r.options.ResumeCfg.Lock() + currentInfo, ok := r.options.ResumeCfg.Current[ip] + if !ok { + currentInfo = &ResumeInfo{} + r.options.ResumeCfg.Current[ip] = currentInfo + } + if currentInfo.InFlight == nil { + currentInfo.InFlight = make(map[uint32]InFlightResume) + } + resumeFromInfo, ok := r.options.ResumeCfg.ResumeFrom[ip] + if !ok { + resumeFromInfo = &ResumeInfo{} + r.options.ResumeCfg.ResumeFrom[ip] = resumeFromInfo } - if r.options.EnableProgressBar { - r.stats.IncrementCounter("packets", 1) + r.options.ResumeCfg.Unlock() + cleanupInFlight := func(current *ResumeInfo, portParam uint32) { + current.Lock() + current.InFlight[portParam] = InFlightResume{ + Completed: true, + } + current.Unlock() + } + skipHandler := func(portParam uint32) bool { + var skip bool + if _, isInFlight := resumeFromInfo.InFlight[portParam]; !isInFlight { // port wasn't found + gologger.Debug().Msgf("[%s] Repeating \"%v\": Resume - Port not found\n", ip, port) + skip = false + } else if item, isInFlight := resumeFromInfo.InFlight[portParam]; isInFlight && !item.Completed { // port scan wasn't completed successfully + gologger.Debug().Msgf("[%s] Repeating \"%v\": Resume - Port scan wasn't completed\n", ip, port) + skip = false + } else if item, isInFlight := resumeFromInfo.InFlight[portParam]; isInFlight && item.Completed { // port scan is already completed + gologger.Debug().Msgf("[%s] Skipping \"%v\": Resume - Port scan already completed\n", ip, port) + skip = true + } + return skip } + // connect scan + go func(portParam uint32) { + currentInfo.Lock() + _, ok := currentInfo.InFlight[portParam] + if !ok { + currentInfo.InFlight[portParam] = InFlightResume{ + Completed: false, + } + } + currentInfo.Unlock() + if skip := skipHandler(portParam); skip { + return + } + handler := func() { + // on scan complete cleanup the inflight data + cleanupInFlight(currentInfo, portParam) + } + if shouldUseRawPackets { + r.RawSocketEnumeration(ip, port, handler) + } else { + r.wgscan.Add() + + go r.handleHostPort(ip, port, handler) + } + if r.options.EnableProgressBar { + r.stats.IncrementCounter("packets", 1) + } + }(uint32(port)) } } @@ -196,6 +249,14 @@ func (r *Runner) RunEnumeration() error { return r.handleNmap() } +func (r *Runner) ShowScanResultOnExit() { + r.handleOutput() + err := r.handleNmap() + if err != nil { + gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) + } +} + // Close runner instance func (r *Runner) Close() { os.RemoveAll(r.targetsFile) @@ -245,9 +306,9 @@ func (r *Runner) BackgroundWorkers() { r.scanner.StartWorkers() } -func (r *Runner) RawSocketEnumeration(ip string, port int) { +func (r *Runner) RawSocketEnumeration(ip string, port int, handler func()) { // skip invalid combinations - r.handleHostPortSyn(ip, port) + r.handleHostPortSyn(ip, port, handler) } // check if an ip can be scanned in case CDN exclusions are enabled @@ -266,7 +327,8 @@ func (r *Runner) canIScanIfCDN(host string, port int) bool { return port == 80 || port == 443 } -func (r *Runner) handleHostPort(host string, port int) { +func (r *Runner) handleHostPort(host string, port int, handler func()) { + defer handler() defer r.wgscan.Done() // performs cdn scan exclusions checks @@ -285,7 +347,8 @@ func (r *Runner) handleHostPort(host string, port int) { } } -func (r *Runner) handleHostPortSyn(host string, port int) { +func (r *Runner) handleHostPortSyn(host string, port int, handler func()) { + defer handler() // performs cdn scan exclusions checks if !r.canIScanIfCDN(host, port) { gologger.Debug().Msgf("Skipping cdn target: %s:%d\n", host, port) From 347377225fdf732e88999b17ea1936e10afe84fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 19:54:30 +0530 Subject: [PATCH 50/67] chore(deps): bump golangci/golangci-lint-action from 2 to 3.1.0 (#273) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 2 to 3.1.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v2...v3.1.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 283820d3..9042fa45 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -16,7 +16,7 @@ jobs: run: sudo apt install libpcap-dev - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3.1.0 with: version: latest args: --timeout 5m From 7e6cc5094d312da0d5b03eaf33e228e7ebf68f96 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 7 Mar 2022 10:53:03 +0100 Subject: [PATCH 51/67] implementing stop/resume using blackrock seed+index --- v2/go.mod | 3 -- v2/go.sum | 4 -- v2/pkg/runner/resume.go | 30 ++++-------- v2/pkg/runner/runner.go | 103 +++++++++++++++++----------------------- 4 files changed, 51 insertions(+), 89 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 55bf305c..41494569 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -26,7 +26,6 @@ require ( require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/stretchr/testify v1.7.0 ) @@ -43,7 +42,6 @@ require ( github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/hmap v0.0.1 // indirect - github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 // indirect @@ -51,7 +49,6 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect go.uber.org/atomic v1.9.0 // indirect golang.org/x/text v0.3.7 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 5c1bc125..fd5e4012 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -88,11 +88,9 @@ github.com/projectdiscovery/dnsx v1.0.7-0.20210927160546-05f957862698 h1:LSE8Fu4 github.com/projectdiscovery/dnsx v1.0.7-0.20210927160546-05f957862698/go.mod h1:KNG4N3G6/QPtv381vQU7lHCJpxz8AvCq7h4pIFDuWwQ= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc h1:dbDgsj26PW06L3zMo7AT08IbEqMd2u8QQ1BvlfMAY2w= github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= -github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -115,7 +113,6 @@ github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vP github.com/projectdiscovery/networkpolicy v0.0.0-20210414200240-b3fa8abf324c/go.mod h1:KZIP5x7t+bH2dASgnYaqbiLI4z0gxXzekwUtarrQMdc= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d h1:tFPD9+3lq+XMHpQ/zMMp41RdXEl90h+KZli7UQt1LNY= github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d/go.mod h1:c5lCypH3Wv7PNRHFsTbmTWlOAV502VlqUmS9A/2wlNU= @@ -216,7 +213,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/v2/pkg/runner/resume.go b/v2/pkg/runner/resume.go index 8afdf11a..e772898b 100644 --- a/v2/pkg/runner/resume.go +++ b/v2/pkg/runner/resume.go @@ -2,12 +2,13 @@ package runner import ( "encoding/json" - "github.com/projectdiscovery/fileutil" - "github.com/projectdiscovery/gologger" "io/ioutil" "os" "path/filepath" "sync" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" ) // Default resume file @@ -24,34 +25,19 @@ func DefaultResumeFilePath() string { // ResumeCfg contains the scan progression type ResumeCfg struct { sync.RWMutex - ResumeFrom map[string]*ResumeInfo `json:"resumeFrom"` - Current map[string]*ResumeInfo `json:"-"` -} - -// InFlightResume -type InFlightResume struct { - Completed bool `json:"completed"` -} - -// ResumeInfo -type ResumeInfo struct { - sync.RWMutex - InFlight map[uint32]InFlightResume `json:"inFlight"` + Retry int `json:"retry"` + Seed int64 `json:"seed"` + Index int64 `json:"index"` } // NewResumeCfg creates a new scan progression structure func NewResumeCfg() *ResumeCfg { - return &ResumeCfg{ - ResumeFrom: make(map[string]*ResumeInfo), - Current: make(map[string]*ResumeInfo), - } + return &ResumeCfg{} } // SaveResumeConfig to file func (resumeCfg *ResumeCfg) SaveResumeConfig() error { - newresumeCfg := NewResumeCfg() - newresumeCfg.ResumeFrom = resumeCfg.Current - data, _ := json.MarshalIndent(newresumeCfg, "", "\t") + data, _ := json.MarshalIndent(resumeCfg, "", "\t") return os.WriteFile(DefaultResumeFilePath(), data, os.ModePerm) } diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 6e71578f..62f186e8 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -153,84 +153,69 @@ func (r *Runner) RunEnumeration() error { shouldUseRawPackets := isOSSupported() && privileges.IsPrivileged && r.options.ScanType == SynScan // Retries are performed regardless of the previous scan results due to network unreliability for currentRetry := 0; currentRetry < r.options.Retries; currentRetry++ { + if currentRetry < r.options.ResumeCfg.Retry { + gologger.Debug().Msgf("Skipping Retry: %d\n", currentRetry) + continue + } + // Use current time as seed - b := blackrock.New(int64(Range), time.Now().UnixNano()) + currentSeed := time.Now().UnixNano() + r.options.ResumeCfg.RLock() + if r.options.ResumeCfg.Seed > 0 { + currentSeed = r.options.ResumeCfg.Seed + } + r.options.ResumeCfg.RUnlock() + + // keep track of current retry and seed for resume + r.options.ResumeCfg.Lock() + r.options.ResumeCfg.Retry = currentRetry + r.options.ResumeCfg.Seed = currentSeed + r.options.ResumeCfg.Unlock() + + b := blackrock.New(int64(Range), currentSeed) for index := int64(0); index < int64(Range); index++ { xxx := b.Shuffle(index) ipIndex := xxx / int64(portsCount) portIndex := int(xxx % int64(portsCount)) ip := r.PickIP(targets, ipIndex) port := r.PickPort(portIndex) + + r.options.ResumeCfg.RLock() + resumeCfgIndex := r.options.ResumeCfg.Index + r.options.ResumeCfg.RUnlock() + if index < resumeCfgIndex { + gologger.Debug().Msgf("Skipping \"%s:%d\": Resume - Port scan already completed\n", ip, port) + continue + } + r.limiter.Take() //resume cfg logic r.options.ResumeCfg.Lock() - currentInfo, ok := r.options.ResumeCfg.Current[ip] - if !ok { - currentInfo = &ResumeInfo{} - r.options.ResumeCfg.Current[ip] = currentInfo - } - if currentInfo.InFlight == nil { - currentInfo.InFlight = make(map[uint32]InFlightResume) - } - resumeFromInfo, ok := r.options.ResumeCfg.ResumeFrom[ip] - if !ok { - resumeFromInfo = &ResumeInfo{} - r.options.ResumeCfg.ResumeFrom[ip] = resumeFromInfo - } + r.options.ResumeCfg.Index = index r.options.ResumeCfg.Unlock() - cleanupInFlight := func(current *ResumeInfo, portParam uint32) { - current.Lock() - current.InFlight[portParam] = InFlightResume{ - Completed: true, - } - current.Unlock() - } - skipHandler := func(portParam uint32) bool { - var skip bool - if _, isInFlight := resumeFromInfo.InFlight[portParam]; !isInFlight { // port wasn't found - gologger.Debug().Msgf("[%s] Repeating \"%v\": Resume - Port not found\n", ip, port) - skip = false - } else if item, isInFlight := resumeFromInfo.InFlight[portParam]; isInFlight && !item.Completed { // port scan wasn't completed successfully - gologger.Debug().Msgf("[%s] Repeating \"%v\": Resume - Port scan wasn't completed\n", ip, port) - skip = false - } else if item, isInFlight := resumeFromInfo.InFlight[portParam]; isInFlight && item.Completed { // port scan is already completed - gologger.Debug().Msgf("[%s] Skipping \"%v\": Resume - Port scan already completed\n", ip, port) - skip = true - } - return skip - } // connect scan go func(portParam uint32) { - currentInfo.Lock() - _, ok := currentInfo.InFlight[portParam] - if !ok { - currentInfo.InFlight[portParam] = InFlightResume{ - Completed: false, - } - } - currentInfo.Unlock() - if skip := skipHandler(portParam); skip { - return - } - handler := func() { - // on scan complete cleanup the inflight data - cleanupInFlight(currentInfo, portParam) - } if shouldUseRawPackets { - r.RawSocketEnumeration(ip, port, handler) + r.RawSocketEnumeration(ip, port) } else { r.wgscan.Add() - go r.handleHostPort(ip, port, handler) + go r.handleHostPort(ip, port) } if r.options.EnableProgressBar { r.stats.IncrementCounter("packets", 1) } }(uint32(port)) } - } - r.wgscan.Wait() + r.wgscan.Wait() + + r.options.ResumeCfg.Lock() + if r.options.ResumeCfg.Seed > 0 { + r.options.ResumeCfg.Seed = 0 + } + r.options.ResumeCfg.Unlock() + } if r.options.WarmUpTime > 0 { time.Sleep(time.Duration(r.options.WarmUpTime) * time.Second) @@ -306,9 +291,9 @@ func (r *Runner) BackgroundWorkers() { r.scanner.StartWorkers() } -func (r *Runner) RawSocketEnumeration(ip string, port int, handler func()) { +func (r *Runner) RawSocketEnumeration(ip string, port int) { // skip invalid combinations - r.handleHostPortSyn(ip, port, handler) + r.handleHostPortSyn(ip, port) } // check if an ip can be scanned in case CDN exclusions are enabled @@ -327,8 +312,7 @@ func (r *Runner) canIScanIfCDN(host string, port int) bool { return port == 80 || port == 443 } -func (r *Runner) handleHostPort(host string, port int, handler func()) { - defer handler() +func (r *Runner) handleHostPort(host string, port int) { defer r.wgscan.Done() // performs cdn scan exclusions checks @@ -347,8 +331,7 @@ func (r *Runner) handleHostPort(host string, port int, handler func()) { } } -func (r *Runner) handleHostPortSyn(host string, port int, handler func()) { - defer handler() +func (r *Runner) handleHostPortSyn(host string, port int) { // performs cdn scan exclusions checks if !r.canIScanIfCDN(host, port) { gologger.Debug().Msgf("Skipping cdn target: %s:%d\n", host, port) From dca9ecde6c13b11b996670c0b5c0d67b7fe23872 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 7 Mar 2022 10:55:47 +0100 Subject: [PATCH 52/67] adding go setup step --- .github/workflows/lint-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 9042fa45..ddb303da 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -12,6 +12,11 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + - name: Install Dependences run: sudo apt install libpcap-dev From 1a2510d5bbdbea7a7bd285a0ce06fb17c10bade2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 01:54:47 +0530 Subject: [PATCH 53/67] chore(deps): bump actions/checkout from 2 to 3 (#279) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-test.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dockerhub-push.yml | 2 +- .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 6 +++--- .github/workflows/sonarcloud.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6dbfcc87..ffc0e2d3 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -18,7 +18,7 @@ jobs: go-version: 1.17 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install libpcap-dev run: sudo apt install libpcap-dev diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fcdd108b..8a61036a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: run: sudo apt install libpcap-dev - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index bc650f75..36e55c96 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Get Github tag id: meta diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 8d1bdcbe..8ecc4b16 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -19,7 +19,7 @@ jobs: go-version: 1.17 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install required packages (libpcap-dev, nmap, simplehttpserver) run: | diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 9042fa45..5af8b76d 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Dependences run: sudo apt install libpcap-dev diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 8937d975..a4082be3 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -10,7 +10,7 @@ jobs: runs-on: macos-latest steps: - name: Code checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Code checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go @@ -55,7 +55,7 @@ jobs: runs-on: windows-latest steps: - name: Code checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 6c046c39..7cf9ef0b 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -13,7 +13,7 @@ jobs: name: SonarCloud runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis From 8ce607205c56d70bcd9c377cc86236c1b33392b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 01:55:00 +0530 Subject: [PATCH 54/67] chore(deps): bump golang from 1.17.7-alpine to 1.17.8-alpine (#280) Bumps golang from 1.17.7-alpine to 1.17.8-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 18313fb2..d0bc439a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.7-alpine AS builder +FROM golang:1.17.8-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From 8bc91bb70bb96da5d2ebcb26ab514dbeab9545c7 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 17 Mar 2022 14:59:04 +0100 Subject: [PATCH 55/67] Prototype of stream support --- v2/go.mod | 2 +- v2/go.sum | 2 + v2/pkg/runner/options.go | 40 ++----- v2/pkg/runner/runner.go | 231 +++++++++++++++++++++----------------- v2/pkg/runner/targets.go | 17 ++- v2/pkg/runner/validate.go | 15 +++ 6 files changed, 174 insertions(+), 133 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 41494569..b1f880b8 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -11,7 +11,7 @@ require ( github.com/projectdiscovery/dnsx v1.0.7-0.20210927160546-05f957862698 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc - github.com/projectdiscovery/goflags v0.0.7 + github.com/projectdiscovery/goflags v0.0.8-0.20220304165250-2530b305a4a9 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/ipranger v0.0.3-0.20210831161617-ac80efae0961 github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 diff --git a/v2/go.sum b/v2/go.sum index fd5e4012..13c893c5 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -94,6 +94,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20220304165250-2530b305a4a9 h1:J05G/rKDM/MSWI3FrXbnCFM7PtZeV+gRic6wzS8eLqI= +github.com/projectdiscovery/goflags v0.0.8-0.20220304165250-2530b305a4a9/go.mod h1:37KhVbVLllyuIAgpXGqcvE/hsFEwJ+ctEUSHawjhsBY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 48fb6bcf..877ef1f5 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -1,9 +1,10 @@ package runner import ( - "github.com/projectdiscovery/fileutil" "os" + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" ) @@ -54,6 +55,7 @@ type Options struct { StatsInterval int // StatsInterval is the number of seconds to display stats after Resume bool ResumeCfg *ResumeCfg + Stream bool } // OnResultCallback (hostname, ip, ports) @@ -66,14 +68,14 @@ func ParseOptions() *Options { flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Naabu is a port scanning tool written in Go that allows you to enumerate open ports for hosts in a fast and reliable manner.`) - createGroup(flagSet, "input", "Input", + flagSet.CreateGroup("input", "Input", flagSet.NormalizedStringSliceVarP(&options.Host, "host", "", []string{}, "hosts to scan ports for (comma-separated)"), flagSet.StringVarP(&options.HostsFile, "l", "list", "", "list of hosts to scan ports (file)"), flagSet.StringVarP(&options.ExcludeIps, "eh", "exclude-hosts", "", "hosts to exclude from the scan (comma-separated)"), flagSet.StringVarP(&options.ExcludeIpsFile, "ef", "exclude-file", "", "list of hosts to exclude from scan (file)"), ) - createGroup(flagSet, "port", "Port", + flagSet.CreateGroup("port", "Port", flagSet.StringVarP(&options.Ports, "p", "port", "", "ports to scan (80,443, 100-200"), flagSet.StringVarP(&options.TopPorts, "tp", "top-ports", "", "top ports to scan (default 100)"), flagSet.StringVarP(&options.ExcludePorts, "ep", "exclude-ports", "", "ports to exclude from scan (comma-separated)"), @@ -81,18 +83,18 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.ExcludeCDN, "ec", "exclude-cdn", false, "skip full port scans for CDN's (only checks for 80,443)"), ) - createGroup(flagSet, "rate-limit", "Rate-limit", + flagSet.CreateGroup("rate-limit", "Rate-limit", flagSet.IntVar(&options.Threads, "c", 25, "general internal worker threads"), flagSet.IntVar(&options.Rate, "rate", DefaultRateSynScan, "packets to send per second"), ) - createGroup(flagSet, "output", "Output", + flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output to (optional)"), flagSet.BoolVar(&options.JSON, "json", false, "write output in JSON lines format"), flagSet.BoolVar(&options.CSV, "csv", false, "write output in csv format"), ) - createGroup(flagSet, "config", "Configuration", + flagSet.CreateGroup("config", "Configuration", flagSet.BoolVarP(&options.ScanAllIPS, "sa", "scan-all-ips", false, "scan all the IP's associated with DNS record"), flagSet.StringVarP(&options.ScanType, "s", "scan-type", SynScan, "type of port scan (SYN/CONNECT)"), flagSet.StringVar(&options.SourceIP, "source-ip", "", "source ip"), @@ -103,9 +105,10 @@ func ParseOptions() *Options { flagSet.StringVar(&options.Resolvers, "r", "", "list of custom resolver dns resolution (comma separated or from file)"), flagSet.StringVar(&options.Proxy, "proxy", "", "socks5 proxy"), flagSet.BoolVar(&options.Resume, "resume", false, "Resume"), + flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode (disables resume, nmap, verify, retries, shuffling, etc)"), ) - createGroup(flagSet, "optimization", "Optimization", + flagSet.CreateGroup("optimization", "Optimization", flagSet.IntVar(&options.Retries, "retries", DefaultRetriesSynScan, "number of retries for the port scan"), flagSet.IntVar(&options.Timeout, "timeout", DefaultPortTimeoutSynScan, "millisecond to wait before timing out"), flagSet.IntVar(&options.WarmUpTime, "warm-up-time", 2, "time in seconds between scan phases"), @@ -113,7 +116,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Verify, "verify", false, "validate the ports again with TCP verification"), ) - createGroup(flagSet, "debug", "Debug", + flagSet.CreateGroup("debug", "Debug", flagSet.BoolVar(&options.Debug, "debug", false, "display debugging information"), flagSet.BoolVarP(&options.Verbose, "v", "verbose", false, "display verbose output"), flagSet.BoolVarP(&options.NoColor, "nc", "no-color", false, "disable colors in CLI output"), @@ -126,7 +129,7 @@ func ParseOptions() *Options { _ = flagSet.Parse() // Check if stdin pipe was given - options.Stdin = hasStdin() + options.Stdin = fileutil.HasStdin() // Read the inputs and configure the logging options.configureOutput() @@ -169,22 +172,3 @@ func ParseOptions() *Options { func (options *Options) ShouldLoadResume() bool { return options.Resume && fileutil.FileExists(DefaultResumeFilePath()) } - -func hasStdin() bool { - stat, err := os.Stdin.Stat() - if err != nil { - return false - } - - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 - isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - - return isPipedFromChrDev || isPipedFromFIFO -} - -func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { - flagSet.SetGroup(groupName, description) - for _, currentFlag := range flags { - currentFlag.Group(groupName) - } -} diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 62f186e8..0a084573 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -27,13 +27,14 @@ import ( // Runner is an instance of the port enumeration // client used to orchestrate the whole process. type Runner struct { - options *Options - targetsFile string - scanner *scan.Scanner - limiter ratelimit.Limiter - wgscan sizedwaitgroup.SizedWaitGroup - dnsclient *dnsx.DNSX - stats *clistats.Statistics + options *Options + targetsFile string + scanner *scan.Scanner + limiter ratelimit.Limiter + wgscan sizedwaitgroup.SizedWaitGroup + dnsclient *dnsx.DNSX + stats *clistats.Statistics + streamChannel chan *net.IPNet } // NewRunner creates a new runner struct instance by parsing @@ -42,6 +43,7 @@ func NewRunner(options *Options) (*Runner, error) { runner := &Runner{ options: options, } + runner.streamChannel = make(chan *net.IPNet) excludedIps, err := parseExcludedIps(options) if err != nil { @@ -114,124 +116,153 @@ func (r *Runner) RunEnumeration() error { r.BackgroundWorkers() } - err := r.Load() - if err != nil { - return err + if r.options.Stream { + go r.Load() + } else { + err := r.Load() + if err != nil { + return err + } } + // Scan workers r.wgscan = sizedwaitgroup.New(r.options.Rate) r.limiter = ratelimit.New(r.options.Rate) - // shrinks the ips to the minimum amount of cidr - var targets []*net.IPNet - r.scanner.IPRanger.Hosts.Scan(func(k, v []byte) error { - targets = append(targets, ipranger.ToCidr(string(k))) - return nil - }) - targets, _ = mapcidr.CoalesceCIDRs(targets) - var targetsCount, portsCount uint64 - for _, target := range targets { - targetsCount += mapcidr.AddressCountIpnet(target) - } - portsCount = uint64(len(r.scanner.Ports)) + shouldUseRawPackets := isOSSupported() && privileges.IsPrivileged && r.options.ScanType == SynScan - r.scanner.State = scan.Scan - Range := targetsCount * portsCount - if r.options.EnableProgressBar { - r.stats.AddStatic("ports", portsCount) - r.stats.AddStatic("hosts", targetsCount) - r.stats.AddStatic("retries", r.options.Retries) - r.stats.AddStatic("startedAt", time.Now()) - r.stats.AddCounter("packets", uint64(0)) - r.stats.AddCounter("errors", uint64(0)) - r.stats.AddCounter("total", Range*uint64(r.options.Retries)) - if err := r.stats.Start(makePrintCallback(), time.Duration(r.options.StatsInterval)*time.Second); err != nil { - gologger.Warning().Msgf("Couldn't start statistics: %s\n", err) + if r.options.Stream { + for cidr := range r.streamChannel { + ipStream, _ := mapcidr.IPAddressesAsStream(cidr.String()) + for ip := range ipStream { + for _, port := range r.scanner.Ports { + r.limiter.Take() + go func(ip string, port int) { + if shouldUseRawPackets { + r.RawSocketEnumeration(ip, port) + } else { + r.wgscan.Add() + + go r.handleHostPort(ip, port) + } + }(ip, port) + } + } } - } - - shouldUseRawPackets := isOSSupported() && privileges.IsPrivileged && r.options.ScanType == SynScan - // Retries are performed regardless of the previous scan results due to network unreliability - for currentRetry := 0; currentRetry < r.options.Retries; currentRetry++ { - if currentRetry < r.options.ResumeCfg.Retry { - gologger.Debug().Msgf("Skipping Retry: %d\n", currentRetry) - continue + r.wgscan.Wait() + r.handleOutput() + return nil + } else { + // shrinks the ips to the minimum amount of cidr + var targets []*net.IPNet + r.scanner.IPRanger.Hosts.Scan(func(k, v []byte) error { + targets = append(targets, ipranger.ToCidr(string(k))) + return nil + }) + targets, _ = mapcidr.CoalesceCIDRs(targets) + var targetsCount, portsCount uint64 + for _, target := range targets { + targetsCount += mapcidr.AddressCountIpnet(target) } - - // Use current time as seed - currentSeed := time.Now().UnixNano() - r.options.ResumeCfg.RLock() - if r.options.ResumeCfg.Seed > 0 { - currentSeed = r.options.ResumeCfg.Seed + portsCount = uint64(len(r.scanner.Ports)) + + r.scanner.State = scan.Scan + Range := targetsCount * portsCount + if r.options.EnableProgressBar { + r.stats.AddStatic("ports", portsCount) + r.stats.AddStatic("hosts", targetsCount) + r.stats.AddStatic("retries", r.options.Retries) + r.stats.AddStatic("startedAt", time.Now()) + r.stats.AddCounter("packets", uint64(0)) + r.stats.AddCounter("errors", uint64(0)) + r.stats.AddCounter("total", Range*uint64(r.options.Retries)) + if err := r.stats.Start(makePrintCallback(), time.Duration(r.options.StatsInterval)*time.Second); err != nil { + gologger.Warning().Msgf("Couldn't start statistics: %s\n", err) + } } - r.options.ResumeCfg.RUnlock() - - // keep track of current retry and seed for resume - r.options.ResumeCfg.Lock() - r.options.ResumeCfg.Retry = currentRetry - r.options.ResumeCfg.Seed = currentSeed - r.options.ResumeCfg.Unlock() - - b := blackrock.New(int64(Range), currentSeed) - for index := int64(0); index < int64(Range); index++ { - xxx := b.Shuffle(index) - ipIndex := xxx / int64(portsCount) - portIndex := int(xxx % int64(portsCount)) - ip := r.PickIP(targets, ipIndex) - port := r.PickPort(portIndex) - r.options.ResumeCfg.RLock() - resumeCfgIndex := r.options.ResumeCfg.Index - r.options.ResumeCfg.RUnlock() - if index < resumeCfgIndex { - gologger.Debug().Msgf("Skipping \"%s:%d\": Resume - Port scan already completed\n", ip, port) + // Retries are performed regardless of the previous scan results due to network unreliability + for currentRetry := 0; currentRetry < r.options.Retries; currentRetry++ { + if currentRetry < r.options.ResumeCfg.Retry { + gologger.Debug().Msgf("Skipping Retry: %d\n", currentRetry) continue } - r.limiter.Take() - //resume cfg logic + // Use current time as seed + currentSeed := time.Now().UnixNano() + r.options.ResumeCfg.RLock() + if r.options.ResumeCfg.Seed > 0 { + currentSeed = r.options.ResumeCfg.Seed + } + r.options.ResumeCfg.RUnlock() + + // keep track of current retry and seed for resume r.options.ResumeCfg.Lock() - r.options.ResumeCfg.Index = index + r.options.ResumeCfg.Retry = currentRetry + r.options.ResumeCfg.Seed = currentSeed r.options.ResumeCfg.Unlock() - // connect scan - go func(portParam uint32) { - if shouldUseRawPackets { - r.RawSocketEnumeration(ip, port) - } else { - r.wgscan.Add() - go r.handleHostPort(ip, port) + b := blackrock.New(int64(Range), currentSeed) + for index := int64(0); index < int64(Range); index++ { + xxx := b.Shuffle(index) + ipIndex := xxx / int64(portsCount) + portIndex := int(xxx % int64(portsCount)) + ip := r.PickIP(targets, ipIndex) + port := r.PickPort(portIndex) + + r.options.ResumeCfg.RLock() + resumeCfgIndex := r.options.ResumeCfg.Index + r.options.ResumeCfg.RUnlock() + if index < resumeCfgIndex { + gologger.Debug().Msgf("Skipping \"%s:%d\": Resume - Port scan already completed\n", ip, port) + continue } - if r.options.EnableProgressBar { - r.stats.IncrementCounter("packets", 1) - } - }(uint32(port)) - } - r.wgscan.Wait() + r.limiter.Take() + //resume cfg logic + r.options.ResumeCfg.Lock() + r.options.ResumeCfg.Index = index + r.options.ResumeCfg.Unlock() + // connect scan + go func(port int) { + if shouldUseRawPackets { + r.RawSocketEnumeration(ip, port) + } else { + r.wgscan.Add() + + go r.handleHostPort(ip, port) + } + if r.options.EnableProgressBar { + r.stats.IncrementCounter("packets", 1) + } + }(port) + } - r.options.ResumeCfg.Lock() - if r.options.ResumeCfg.Seed > 0 { - r.options.ResumeCfg.Seed = 0 + r.wgscan.Wait() + + r.options.ResumeCfg.Lock() + if r.options.ResumeCfg.Seed > 0 { + r.options.ResumeCfg.Seed = 0 + } + r.options.ResumeCfg.Unlock() } - r.options.ResumeCfg.Unlock() - } - if r.options.WarmUpTime > 0 { - time.Sleep(time.Duration(r.options.WarmUpTime) * time.Second) - } + if r.options.WarmUpTime > 0 { + time.Sleep(time.Duration(r.options.WarmUpTime) * time.Second) + } - r.scanner.State = scan.Done + r.scanner.State = scan.Done - // Validate the hosts if the user has asked for second step validation - if r.options.Verify { - r.ConnectVerification() - } + // Validate the hosts if the user has asked for second step validation + if r.options.Verify { + r.ConnectVerification() + } - r.handleOutput() + r.handleOutput() - // handle nmap - return r.handleNmap() + // handle nmap + return r.handleNmap() + } } func (r *Runner) ShowScanResultOnExit() { diff --git a/v2/pkg/runner/targets.go b/v2/pkg/runner/targets.go index 37bdd138..41d5a178 100644 --- a/v2/pkg/runner/targets.go +++ b/v2/pkg/runner/targets.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ipranger" + "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/naabu/v2/pkg/privileges" "github.com/projectdiscovery/naabu/v2/pkg/scan" "github.com/remeh/sizedwaitgroup" @@ -79,6 +80,9 @@ func (r *Runner) mergeToFile() (string, error) { } func (r *Runner) PreProcessTargets() error { + if r.options.Stream { + defer close(r.streamChannel) + } wg := sizedwaitgroup.New(r.options.Threads) f, err := os.Open(r.targetsFile) if err != nil { @@ -105,12 +109,15 @@ func (r *Runner) AddTarget(target string) error { if target == "" { return nil } else if ipranger.IsCidr(target) { - // Add cidr directly to ranger, as single ips would allocate more resources later - if err := r.scanner.IPRanger.AddHostWithMetadata(target, "cidr"); err != nil { + if r.options.Stream { + r.streamChannel <- iputil.ToCidr(target) + } else if err := r.scanner.IPRanger.AddHostWithMetadata(target, "cidr"); err != nil { // Add cidr directly to ranger, as single ips would allocate more resources later gologger.Warning().Msgf("%s\n", err) } } else if ipranger.IsIP(target) && !r.scanner.IPRanger.Contains(target) { - if err := r.scanner.IPRanger.AddHostWithMetadata(target, "ip"); err != nil { + if r.options.Stream { + r.streamChannel <- iputil.ToCidr(target) + } else if err := r.scanner.IPRanger.AddHostWithMetadata(target, "ip"); err != nil { gologger.Warning().Msgf("%s\n", err) } } else { @@ -119,7 +126,9 @@ func (r *Runner) AddTarget(target string) error { return err } for _, ip := range ips { - if err := r.scanner.IPRanger.AddHostWithMetadata(ip, target); err != nil { + if r.options.Stream { + r.streamChannel <- iputil.ToCidr(ip) + } else if err := r.scanner.IPRanger.AddHostWithMetadata(ip, target); err != nil { gologger.Warning().Msgf("%s\n", err) } } diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index 085e4130..14e2ab45 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -75,6 +75,21 @@ func (options *Options) validateOptions() error { } } + if options.Stream { + if options.Resume { + return errors.New("resume not supported in stream mode") + } + if options.EnableProgressBar { + return errors.New("stats not supported in stream mode") + } + if options.Verify { + return errors.New("verify not supported in stream mode") + } + if options.Nmap { + return errors.New("nmap not supported in stream mode") + } + } + return nil } From 0270f957a87bf7d8f1ed2d5993e335d5e4e95d76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 13:06:18 +0000 Subject: [PATCH 56/67] chore(deps): bump golang from 1.17.8-alpine to 1.18.0-alpine Bumps golang from 1.17.8-alpine to 1.18.0-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d0bc439a..cf04ed1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.8-alpine AS builder +FROM golang:1.18.0-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest From 2af52fb96ebe365349813f387931f5e6736bbf15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 13:06:22 +0000 Subject: [PATCH 57/67] chore(deps): bump alpine from 3.15.0 to 3.15.1 Bumps alpine from 3.15.0 to 3.15.1. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d0bc439a..5871022a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.8-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest -FROM alpine:3.15.0 +FROM alpine:3.15.1 RUN apk add nmap libpcap-dev bind-tools ca-certificates COPY --from=builder /go/bin/naabu /usr/local/bin/naabu ENTRYPOINT ["naabu"] From bfc5f5469129eba569cbbe346575eb10b13a967e Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 21 Mar 2022 21:14:37 +0100 Subject: [PATCH 58/67] fixing lint issues --- v2/pkg/runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 0a084573..6c7f4357 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -117,7 +117,7 @@ func (r *Runner) RunEnumeration() error { } if r.options.Stream { - go r.Load() + go r.Load() //nolint } else { err := r.Load() if err != nil { From 730e0150ecaa7746fe4c60e85990686dafa8f67e Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 15:53:18 +0100 Subject: [PATCH 59/67] Updating cdncheck --- v2/go.mod | 6 +++--- v2/go.sum | 10 ++++++---- v2/pkg/runner/runner.go | 2 +- v2/pkg/scan/cdn.go | 7 ++++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 41494569..4d6e8c79 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,7 +6,7 @@ require ( github.com/google/gopacket v1.1.19 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e - github.com/projectdiscovery/cdncheck v0.0.2 + github.com/projectdiscovery/cdncheck v0.0.4-0.20220322144854-b2d8ce308abb github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/dnsx v1.0.7-0.20210927160546-05f957862698 github.com/projectdiscovery/fdmax v0.0.3 @@ -35,11 +35,11 @@ require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/json-iterator/go v1.1.11 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/miekg/dns v1.1.43 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/hmap v0.0.1 // indirect github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d // indirect diff --git a/v2/go.sum b/v2/go.sum index fd5e4012..7c4f84c4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -36,8 +36,9 @@ github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= @@ -57,8 +58,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -80,8 +82,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/cdncheck v0.0.2 h1:ZAaezMvVioC/8fC8iilylGT4FBBryzQdXHow+lHnPOE= -github.com/projectdiscovery/cdncheck v0.0.2/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= +github.com/projectdiscovery/cdncheck v0.0.4-0.20220322144854-b2d8ce308abb h1:Q7tb/p2ts+dT+v4nQpxTR0DSng2C1Hlnrw/NE4tgGZw= +github.com/projectdiscovery/cdncheck v0.0.4-0.20220322144854-b2d8ce308abb/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/dnsx v1.0.7-0.20210927160546-05f957862698 h1:LSE8Fu47mp1VcuQ2tb7K2H3UpWdSt/OvxpPiQ7+yC/k= diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 62f186e8..e94369c9 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -304,7 +304,7 @@ func (r *Runner) canIScanIfCDN(host string, port int) bool { } // if exclusion is enabled, but the ip is not part of the CDN ips range we can scan - if ok, err := r.scanner.CdnCheck(host); err == nil && !ok { + if ok, _, err := r.scanner.CdnCheck(host); err == nil && !ok { return true } diff --git a/v2/pkg/scan/cdn.go b/v2/pkg/scan/cdn.go index e4bf1e50..620fd5a9 100644 --- a/v2/pkg/scan/cdn.go +++ b/v2/pkg/scan/cdn.go @@ -1,14 +1,15 @@ package scan import ( - "fmt" "net" + + "github.com/pkg/errors" ) // CdnCheck verifies if the given ip is part of Cdn ranges -func (s *Scanner) CdnCheck(ip string) (bool, error) { +func (s *Scanner) CdnCheck(ip string) (bool, string, error) { if s.cdn == nil { - return false, fmt.Errorf("cdn client not initialized") + return false, "", errors.New("cdn client not initialized") } return s.cdn.Check(net.ParseIP((ip))) } From 67340a1929b474d285b7e721f8baa1b3baaa3ecc Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 15:56:04 +0100 Subject: [PATCH 60/67] fixing test case --- v2/pkg/scan/cdn_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/scan/cdn_test.go b/v2/pkg/scan/cdn_test.go index 02272ab6..e6f94d0d 100644 --- a/v2/pkg/scan/cdn_test.go +++ b/v2/pkg/scan/cdn_test.go @@ -20,7 +20,7 @@ func TestCdnCheck(t *testing.T) { } for _, tt := range tests { t.Run(tt.args, func(t *testing.T) { - isCdn, err := s.CdnCheck(tt.args) + isCdn, _, err := s.CdnCheck(tt.args) if tt.wantErr { assert.NotNil(t, err) } else { From bcdb770eb317c0eb4f52e04bc95355e9928076f9 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 16:09:56 +0100 Subject: [PATCH 61/67] adding ip check --- v2/pkg/scan/cdn.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/v2/pkg/scan/cdn.go b/v2/pkg/scan/cdn.go index 620fd5a9..5a78d4fb 100644 --- a/v2/pkg/scan/cdn.go +++ b/v2/pkg/scan/cdn.go @@ -4,6 +4,7 @@ import ( "net" "github.com/pkg/errors" + "github.com/projectdiscovery/iputil" ) // CdnCheck verifies if the given ip is part of Cdn ranges @@ -11,5 +12,8 @@ func (s *Scanner) CdnCheck(ip string) (bool, string, error) { if s.cdn == nil { return false, "", errors.New("cdn client not initialized") } + if !iputil.IsIP(ip) { + return false, "", errors.Errorf("%s is not a valid ip", ip) + } return s.cdn.Check(net.ParseIP((ip))) } From 39f021b556d465095747e0597ca27e7a4e88744d Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 23 Mar 2022 17:08:52 +0100 Subject: [PATCH 62/67] skipping rate limit for analysis operations --- v2/pkg/runner/runner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index e94369c9..c01f95ad 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -188,7 +188,6 @@ func (r *Runner) RunEnumeration() error { continue } - r.limiter.Take() //resume cfg logic r.options.ResumeCfg.Lock() r.options.ResumeCfg.Index = index @@ -325,6 +324,7 @@ func (r *Runner) handleHostPort(host string, port int) { return } + r.limiter.Take() open, err := r.scanner.ConnectPort(host, port, time.Duration(r.options.Timeout)*time.Millisecond) if open && err == nil { r.scanner.ScanResults.AddPort(host, port) @@ -338,6 +338,7 @@ func (r *Runner) handleHostPortSyn(host string, port int) { return } + r.limiter.Take() r.scanner.EnqueueTCP(host, port, scan.SYN) } From ed91d372db1cb0219e63991a87baf64bfb89fad7 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 23 Mar 2022 21:55:29 +0100 Subject: [PATCH 63/67] adding stream mode for output --- v2/pkg/runner/runner.go | 3 +++ v2/pkg/scan/option.go | 1 + v2/pkg/scan/scan.go | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 57c1cec5..a3cb360e 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -58,6 +58,7 @@ func NewRunner(options *Options) (*Runner, error) { ExcludeCdn: options.ExcludeCDN, ExcludedIps: excludedIps, Proxy: options.Proxy, + Stream: options.Stream, }) if err != nil { return nil, err @@ -132,7 +133,9 @@ func (r *Runner) RunEnumeration() error { shouldUseRawPackets := isOSSupported() && privileges.IsPrivileged && r.options.ScanType == SynScan if r.options.Stream { + r.scanner.State = scan.Scan for cidr := range r.streamChannel { + r.scanner.IPRanger.Add(cidr.String()) ipStream, _ := mapcidr.IPAddressesAsStream(cidr.String()) for ip := range ipStream { for _, port := range r.scanner.Ports { diff --git a/v2/pkg/scan/option.go b/v2/pkg/scan/option.go index ef92be31..ceec6443 100644 --- a/v2/pkg/scan/option.go +++ b/v2/pkg/scan/option.go @@ -13,4 +13,5 @@ type Options struct { ExcludeCdn bool ExcludedIps []string Proxy string + Stream bool } diff --git a/v2/pkg/scan/scan.go b/v2/pkg/scan/scan.go index e68a6c8f..689f8359 100644 --- a/v2/pkg/scan/scan.go +++ b/v2/pkg/scan/scan.go @@ -75,6 +75,7 @@ type Scanner struct { serializeOptions gopacket.SerializeOptions debug bool handlers interface{} + stream bool } // PkgSend is a TCP package @@ -154,6 +155,8 @@ func NewScanner(options *Options) (*Scanner, error) { scanner.proxyDialer = proxyDialer } + scanner.stream = options.Stream + return scanner, nil } @@ -258,7 +261,7 @@ func (s *Scanner) ICMPReadWorker() { // TCPResultWorker handles probes and scan results func (s *Scanner) TCPResultWorker() { for ip := range s.tcpChan { - if s.State == Scan { + if s.State == Scan || s.stream { gologger.Debug().Msgf("Received TCP scan response from %s:%d\n", ip.ip, ip.port) s.ScanResults.AddPort(ip.ip, ip.port) } From ef8cfed346d7cbd90c9067ae305e25a91f12ee28 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 26 Mar 2022 18:11:41 +0100 Subject: [PATCH 64/67] fixing lint errors --- v2/pkg/runner/runner.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index a3cb360e..237355ce 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -135,7 +135,9 @@ func (r *Runner) RunEnumeration() error { if r.options.Stream { r.scanner.State = scan.Scan for cidr := range r.streamChannel { - r.scanner.IPRanger.Add(cidr.String()) + if err := r.scanner.IPRanger.Add(cidr.String()); err != nil { + gologger.Warning().Msgf("Couldn't track %s in scan results: %s\n", cidr, err) + } ipStream, _ := mapcidr.IPAddressesAsStream(cidr.String()) for ip := range ipStream { for _, port := range r.scanner.Ports { From 9fc42bba0519ca2ae221fab7a5eab5d1ba0ca3b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 13:05:09 +0000 Subject: [PATCH 65/67] chore(deps): bump alpine from 3.15.1 to 3.15.2 Bumps alpine from 3.15.1 to 3.15.2. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dc01bd87..6183d082 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.0-alpine AS builder RUN apk add build-base libpcap-dev RUN go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest -FROM alpine:3.15.1 +FROM alpine:3.15.2 RUN apk add nmap libpcap-dev bind-tools ca-certificates COPY --from=builder /go/bin/naabu /usr/local/bin/naabu ENTRYPOINT ["naabu"] From 30caa1d3717bf6ade9f4228bc5a86dd9cf9f1771 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 31 Mar 2022 16:20:57 +0530 Subject: [PATCH 66/67] version update --- v2/pkg/runner/banners.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 7feeaf0f..356cdfde 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -13,11 +13,11 @@ const banner = ` __ ___ ___ ___ _/ / __ __ / _ \/ _ \/ _ \/ _ \/ // / -/_//_/\_,_/\_,_/_.__/\_,_/ v2.0.5 +/_//_/\_,_/\_,_/_.__/\_,_/ v2.0.6 ` // Version is the current version of naabu -const Version = `2.0.5` +const Version = `2.0.6` // showBanner is used to show the banner to the user func showBanner() { From ac606dbe055df13b848c2b58c1b1ca1c343c20a5 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 31 Mar 2022 16:38:00 +0530 Subject: [PATCH 67/67] readme updates --- README.md | 16 ++++++++++------ v2/pkg/runner/options.go | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 93ba3c60..5d4db252 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ RATE-LIMIT: OUTPUT: -o, -output string file to write output to (optional) -json write output in JSON lines format + -csv write output in csv format CONFIGURATION: -scan-all-ips, -sa scan all the IP's associated with DNS record @@ -83,6 +84,8 @@ CONFIGURATION: -nmap-cli string nmap command to run on found results (example: -nmap-cli 'nmap -sV') -r string list of custom resolver dns resolution (comma separated or from file) -proxy string socks5 proxy + -resume resume scan using resume.cfg + -stream stream mode (disables resume, nmap, verify, retries, shuffling, etc) OPTIMIZATION: -retries int number of retries for the port scan (default 3) @@ -92,12 +95,13 @@ OPTIMIZATION: -verify validate the ports again with TCP verification DEBUG: - -debug display debugging information - -verbose, -v display verbose output - -no-color, -nc disable colors in CLI output - -silent display only results in output - -version display version of naabu - -stats display stats of the running scan + -debug display debugging information + -verbose, -v display verbose output + -no-color, -nc disable colors in CLI output + -silent display only results in output + -version display version of naabu + -stats display stats of the running scan + -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) ``` # Installation Instructions diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 877ef1f5..3e07307b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -104,8 +104,8 @@ func ParseOptions() *Options { flagSet.StringVar(&options.NmapCLI, "nmap-cli", "", "nmap command to run on found results (example: -nmap-cli 'nmap -sV')"), flagSet.StringVar(&options.Resolvers, "r", "", "list of custom resolver dns resolution (comma separated or from file)"), flagSet.StringVar(&options.Proxy, "proxy", "", "socks5 proxy"), - flagSet.BoolVar(&options.Resume, "resume", false, "Resume"), - flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode (disables resume, nmap, verify, retries, shuffling, etc)"), + flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"), + flagSet.BoolVar(&options.Stream, "stream", false, "stream mode (disables resume, nmap, verify, retries, shuffling, etc)"), ) flagSet.CreateGroup("optimization", "Optimization",