Fix progress with patterns and backup discovery (#548)

* Account for patterns in total tries

Specifying patterns for the word list will no longer cause progress to
go past 100%. Additionally, the GobusterDir transformations for file
extensions and backups will be applied after pattern expansion.

Fixes #405, #480, and #533

* Run backup detection on success only

This is done by re-arranging the code from exiting on channel close to
using the contexts and the results counters to signal the end of work
processing. A little more work is needed to prevent infinite loops
caused by devious services/misconfiguration and to expose to the cli the
ability to have patterns run on successful finds.

Fixes #298

* Prioritize stopping when the context is done

From the Go spec:
> If one or more of the communications can proceed, a single one that
> can proceed is chosen via a uniform pseudo-random selection.

Previously, this meant that some indeterminate amount of work could have
been completed after the context's cancel function had been called.

* Prevent recursion of discovery guess

Successful guesses from the wordlist or a pattern will have discovery
patterns generated based on them and successful discovery guesses will
not. Further processing should require human curation to avoid
automatically generating an unbounded amount of traffic.

Also fixes reading the wordlist from standard in by making it more like
reading from a file now that we have dynamic progress updates.

* Add option for arbitrary discovery patterns

* Test only patterns if provided a pattern file

* Update discover pattern option description

README.md

@@ -185,25 +185,26 @@ USAGE:
    gobuster dns [command options] [arguments...]
 
 OPTIONS:
-   --domain value, --do value           The target domain
-   --show-ips, -i                       Show IP addresses of found domains (default: false)
-   --check-cname, -c                    Also check CNAME records (default: false)
-   --timeout value, --to value          DNS resolver timeout (default: 1s)
-   --wildcard, --wc                     Force continued operation when wildcard found (default: false)
-   --no-fqdn, --nf                      Do not automatically add a trailing dot to the domain, so the resolver uses the DNS search domain (default: false)
-   --resolver value                     Use custom DNS server (format server.com or server.com:port)
-   --wordlist value, -w value           Path to the wordlist. Set to - to use STDIN.
-   --delay value, -d value              Time each thread waits between requests (e.g. 1500ms) (default: 0s)
-   --threads value, -t value            Number of concurrent threads (default: 10)
-   --wordlist-offset value, --wo value  Resume from a given position in the wordlist (default: 0)
-   --output value, -o value             Output file to write results to (defaults to stdout)
-   --quiet, -q                          Don't print the banner and other noise (default: false)
-   --no-progress, --np                  Don't display progress (default: false)
-   --no-error, --ne                     Don't display errors (default: false)
-   --pattern value, -p value            File containing replacement patterns
-   --no-color, --nc                     Disable color output (default: false)
-   --debug                              enable debug output (default: false)
-   --help, -h                           show help
+   --domain value, --do value            The target domain
+   --show-ips, -i                        Show IP addresses of found domains (default: false)
+   --check-cname, -c                     Also check CNAME records (default: false)
+   --timeout value, --to value           DNS resolver timeout (default: 1s)
+   --wildcard, --wc                      Force continued operation when wildcard found (default: false)
+   --no-fqdn, --nf                       Do not automatically add a trailing dot to the domain, so the resolver uses the DNS search domain (default: false)
+   --resolver value                      Use custom DNS server (format server.com or server.com:port)
+   --wordlist value, -w value            Path to the wordlist. Set to - to use STDIN.
+   --delay value, -d value               Time each thread waits between requests (e.g. 1500ms) (default: 0s)
+   --threads value, -t value             Number of concurrent threads (default: 10)
+   --wordlist-offset value, --wo value   Resume from a given position in the wordlist (default: 0)
+   --output value, -o value              Output file to write results to (defaults to stdout)
+   --quiet, -q                           Don't print the banner and other noise (default: false)
+   --no-progress, --np                   Don't display progress (default: false)
+   --no-error, --ne                      Don't display errors (default: false)
+   --pattern value, -p value             File containing replacement patterns
+   --discover-pattern value, --pd value  File containing replacement patterns applied to successful guesses
+   --no-color, --nc                      Disable color output (default: false)
+   --debug                               enable debug output (default: false)
+   --help, -h                            show help
 ```
 
 ### Examples
@@ -401,6 +402,7 @@ OPTIONS:
    --no-progress, --np                                      Don't display progress (default: false)
    --no-error, --ne                                         Don't display errors (default: false)
    --pattern value, -p value                                File containing replacement patterns
+   --discover-pattern value, --pd value                     File containing replacement patterns applied to successful guesses
    --no-color, --nc                                         Disable color output (default: false)
    --debug                                                  enable debug output (default: false)
    --status-codes value, -s value                           Positive status codes (will be overwritten with status-codes-blacklist if set). Can also handle ranges like 200,300-400,404
@@ -411,7 +413,7 @@ OPTIONS:
    --no-status, -n                                          Don't print status codes (default: false)
    --hide-length, --hl                                      Hide the length of the body in the output (default: false)
    --add-slash, -f                                          Append / to each request (default: false)
-   --discover-backup, --db                                  Also search for backup files by appending multiple backup extensions (default: false)
+   --discover-backup, --db                                  Upon finding a file search for backup files by appending multiple backup extensions (default: false)
    --exclude-length value, --xl value                       exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206
    --help, -h                                               show help
 ```
@@ -589,6 +591,7 @@ OPTIONS:
    --no-progress, --np                                      Don't display progress (default: false)
    --no-error, --ne                                         Don't display errors (default: false)
    --pattern value, -p value                                File containing replacement patterns
+   --discover-pattern value, --pd value                     File containing replacement patterns applied to successful guesses
    --no-color, --nc                                         Disable color output (default: false)
    --debug                                                  enable debug output (default: false)
    --append-domain, --ad                                    Append main domain from URL to words from wordlist. Otherwise the fully qualified domains need to be specified in the wordlist. (default: false)
@@ -669,6 +672,7 @@ OPTIONS:
    --no-progress, --np                                      Don't display progress (default: false)
    --no-error, --ne                                         Don't display errors (default: false)
    --pattern value, -p value                                File containing replacement patterns
+   --discover-pattern value, --pd value                     File containing replacement patterns applied to successful guesses
    --no-color, --nc                                         Disable color output (default: false)
    --debug                                                  enable debug output (default: false)
    --exclude-statuscodes value, -b value                    Excluded status codes. Can also handle ranges like 200,300-400,404.
@@ -706,6 +710,7 @@ OPTIONS:
    --no-progress, --np                               Don't display progress (default: false)
    --no-error, --ne                                  Don't display errors (default: false)
    --pattern value, -p value                         File containing replacement patterns
+   --discover-pattern value, --pd value              File containing replacement patterns applied to successful guesses
    --no-color, --nc                                  Disable color output (default: false)
    --debug                                           enable debug output (default: false)
    --useragent value, -a value                       Set the User-Agent string (default: "gobuster/3.7")
@@ -751,6 +756,7 @@ OPTIONS:
    --no-progress, --np                               Don't display progress (default: false)
    --no-error, --ne                                  Don't display errors (default: false)
    --pattern value, -p value                         File containing replacement patterns
+   --discover-pattern value, --pd value              File containing replacement patterns applied to successful guesses
    --no-color, --nc                                  Disable color output (default: false)
    --debug                                           enable debug output (default: false)
    --useragent value, -a value                       Set the User-Agent string (default: "gobuster/3.7")
@@ -785,20 +791,21 @@ USAGE:
    gobuster tftp [command options] [arguments...]
 
 OPTIONS:
-   --server value, -s value             The target TFTP server
-   --timeout value, --to value          TFTP timeout (default: 1s)
-   --wordlist value, -w value           Path to the wordlist. Set to - to use STDIN.
-   --delay value, -d value              Time each thread waits between requests (e.g. 1500ms) (default: 0s)
-   --threads value, -t value            Number of concurrent threads (default: 10)
-   --wordlist-offset value, --wo value  Resume from a given position in the wordlist (default: 0)
-   --output value, -o value             Output file to write results to (defaults to stdout)
-   --quiet, -q                          Don't print the banner and other noise (default: false)
-   --no-progress, --np                  Don't display progress (default: false)
-   --no-error, --ne                     Don't display errors (default: false)
-   --pattern value, -p value            File containing replacement patterns
-   --no-color, --nc                     Disable color output (default: false)
-   --debug                              enable debug output (default: false)
-   --help, -h                           show help
+   --server value, -s value              The target TFTP server
+   --timeout value, --to value           TFTP timeout (default: 1s)
+   --wordlist value, -w value            Path to the wordlist. Set to - to use STDIN.
+   --delay value, -d value               Time each thread waits between requests (e.g. 1500ms) (default: 0s)
+   --threads value, -t value             Number of concurrent threads (default: 10)
+   --wordlist-offset value, --wo value   Resume from a given position in the wordlist (default: 0)
+   --output value, -o value              Output file to write results to (defaults to stdout)
+   --quiet, -q                           Don't print the banner and other noise (default: false)
+   --no-progress, --np                   Don't display progress (default: false)
+   --no-error, --ne                      Don't display errors (default: false)
+   --pattern value, -p value             File containing replacement patterns
+   --discover-pattern value, --pd value  File containing replacement patterns applied to successful guesses
+   --no-color, --nc                      Disable color output (default: false)
+   --debug                               enable debug output (default: false)
+   --help, -h                            show help
 ```
 
 ### Examples
@@ -823,6 +830,7 @@ Note: If the `-w` option is specified at the same time as piping from STDIN, an
 You can supply pattern files that will be applied to every word from the wordlist.
 Just place the string `{GOBUSTER}` in it and this will be replaced with the word.
 This feature is also handy in s3 mode to pre- or postfix certain patterns.
+When supplying patterns, words from the wordlist will not be tried by themselves. If you wish to have patterns and plain words from the wordlist, place `{GOBUSTER}` on a line by itself in the pattern file.
 
 **Caution:** Using a big pattern file can cause a lot of request as every pattern is applied to every word in the wordlist.
 

cli/dir/dir.go

@@ -33,7 +33,7 @@ func getFlags() []cli.Flag {
 		&cli.BoolFlag{Name: "no-status", Aliases: []string{"n"}, Value: false, Usage: "Don't print status codes"},
 		&cli.BoolFlag{Name: "hide-length", Aliases: []string{"hl"}, Value: false, Usage: "Hide the length of the body in the output"},
 		&cli.BoolFlag{Name: "add-slash", Aliases: []string{"f"}, Value: false, Usage: "Append / to each request"},
-		&cli.BoolFlag{Name: "discover-backup", Aliases: []string{"db"}, Value: false, Usage: "Also search for backup files by appending multiple backup extensions"},
+		&cli.BoolFlag{Name: "discover-backup", Aliases: []string{"db"}, Value: false, Usage: "Upon finding a file search for backup files by appending multiple backup extensions"},
 		&cli.StringFlag{Name: "exclude-length", Aliases: []string{"xl"}, Usage: "exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206"},
 	}...)
 	return flags

cli/gobuster.go

@@ -93,14 +93,8 @@ func messageWorker(g *libgobuster.Gobuster, wg *sync.WaitGroup) {
 func printProgress(g *libgobuster.Gobuster) {
 	requestsIssued := g.Progress.RequestsIssued()
 	requestsExpected := g.Progress.RequestsExpected()
-	if g.Opts.Wordlist == "-" {
-		s := fmt.Sprintf("%sProgress: %d", TERMINAL_CLEAR_LINE, requestsIssued)
-		_, _ = fmt.Fprint(os.Stderr, s)
-		// only print status if we already read in the wordlist
-	} else if requestsExpected > 0 {
-		s := fmt.Sprintf("%sProgress: %d / %d (%3.2f%%)", TERMINAL_CLEAR_LINE, requestsIssued, requestsExpected, float32(requestsIssued)*100.0/float32(requestsExpected))
-		_, _ = fmt.Fprint(os.Stderr, s)
-	}
+	s := fmt.Sprintf("%sProgress: %d / %d (%3.2f%%)", TERMINAL_CLEAR_LINE, requestsIssued, requestsExpected, float32(requestsIssued)*100.0/float32(requestsExpected))
+	_, _ = fmt.Fprint(os.Stderr, s)
 }
 
 // progressWorker outputs the progress every tick. It will stop once cancel() is called

cli/options.go

@@ -213,6 +213,7 @@ func GlobalOptions() []cli.Flag {
 		&cli.BoolFlag{Name: "no-progress", Aliases: []string{"np"}, Value: false, Usage: "Don't display progress"},
 		&cli.BoolFlag{Name: "no-error", Aliases: []string{"ne"}, Value: false, Usage: "Don't display errors"},
 		&cli.StringFlag{Name: "pattern", Aliases: []string{"p"}, Usage: "File containing replacement patterns"},
+		&cli.StringFlag{Name: "discover-pattern", Aliases: []string{"pd"}, Usage: "File containing replacement patterns applied to successful guesses"},
 		&cli.BoolFlag{Name: "no-color", Aliases: []string{"nc"}, Value: false, Usage: "Disable color output"},
 		&cli.BoolFlag{Name: "debug", Value: false, Usage: "enable debug output"},
 	}
@@ -261,6 +262,26 @@ func ParseGlobalOptions(c *cli.Context) (libgobuster.Options, error) {
 		}
 	}
 
+	opts.DiscoverPatternFile = c.String("discover-pattern")
+	if opts.DiscoverPatternFile != "" {
+		if _, err := os.Stat(opts.PatternFile); os.IsNotExist(err) {
+			return opts, fmt.Errorf("discover pattern file %q does not exist: %w", opts.DiscoverPatternFile, err)
+		}
+		discoverPatternFile, err := os.Open(opts.DiscoverPatternFile)
+		if err != nil {
+			return opts, fmt.Errorf("could not open discover pattern file %q: %w", opts.DiscoverPatternFile, err)
+		}
+		defer discoverPatternFile.Close()
+
+		scanner := bufio.NewScanner(discoverPatternFile)
+		for scanner.Scan() {
+			opts.DiscoverPatterns = append(opts.DiscoverPatterns, scanner.Text())
+		}
+		if err := scanner.Err(); err != nil {
+			return opts, fmt.Errorf("could not read discover pattern file %q: %w", opts.DiscoverPatternFile, err)
+		}
+	}
+
 	if c.Bool("no-color") {
 		color.NoColor = true
 	}

gobusterdir/gobusterdir.go

@@ -157,42 +157,43 @@ func (d *GobusterDir) PreRun(ctx context.Context, _ *libgobuster.Progress) error
 	return nil
 }
 
-func getBackupFilenames(word string) []string {
-	ret := make([]string, len(backupExtensions)+len(backupDotExtensions))
-	i := 0
-	for _, b := range backupExtensions {
-		ret[i] = fmt.Sprintf("%s%s", word, b)
-		i++
-	}
-	for _, b := range backupDotExtensions {
-		ret[i] = fmt.Sprintf(".%s%s", word, b)
-		i++
+func (d *GobusterDir) AdditionalSuccessWords(word string) []string {
+	if d.options.DiscoverBackup {
+		ret := make([]string, len(backupExtensions)+len(backupDotExtensions))
+		i := 0
+		for _, b := range backupExtensions {
+			ret[i] = fmt.Sprintf("%s%s", word, b)
+			i++
+		}
+		for _, b := range backupDotExtensions {
+			ret[i] = fmt.Sprintf(".%s%s", word, b)
+			i++
+		}
+
+		return ret
 	}
 
-	return ret
+	return []string{}
+}
+
+func (d *GobusterDir) AdditionalWordsLen() int {
+	return len(d.options.ExtensionsParsed.Set)
 }
 
 func (d *GobusterDir) AdditionalWords(word string) []string {
-	var words []string
+	words := make([]string, 0, d.AdditionalWordsLen())
 	// build list of urls to check
 	//   1: No extension
 	//   2: With extension
-	//   3: backupextension
-	if d.options.DiscoverBackup {
-		words = append(words, getBackupFilenames(word)...)
-	}
 	for ext := range d.options.ExtensionsParsed.Set {
 		filename := fmt.Sprintf("%s.%s", word, ext)
 		words = append(words, filename)
-		if d.options.DiscoverBackup {
-			words = append(words, getBackupFilenames(filename)...)
-		}
 	}
 	return words
 }
 
 // ProcessWord is the process implementation of gobusterdir
-func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error {
+func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) (libgobuster.Result, error) {
 	suffix := ""
 	if d.options.UseSlash {
 		suffix = "/"
@@ -243,13 +244,13 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li
 				continue
 			} else {
 				if errors.Is(err, io.EOF) {
-					return libgobuster.ErrorEOF
+					return nil, libgobuster.ErrorEOF
 				} else if os.IsTimeout(err) {
-					return libgobuster.ErrorTimeout
+					return nil, libgobuster.ErrorTimeout
 				} else if errors.Is(err, syscall.ECONNREFUSED) {
-					return libgobuster.ErrorConnectionRefused
+					return nil, libgobuster.ErrorConnectionRefused
 				}
-				return err
+				return nil, err
 			}
 		}
 		break
@@ -267,7 +268,7 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li
 				resultStatus = true
 			}
 		} else {
-			return fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen")
+			return nil, fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen")
 		}
 
 		if resultStatus && !d.options.ExcludeLengthParsed.Contains(int(size)) {
@@ -289,11 +290,11 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li
 			if !d.options.HideLength {
 				r.Size = size
 			}
-			progress.ResultChan <- r
+			return r, nil
 		}
 	}
 
-	return nil
+	return nil, nil
 }
 
 // GetConfigString returns the string representation of the current config

gobusterdir/gobusterdir_test.go

@@ -0,0 +1,34 @@
+package gobusterdir
+
+import (
+	"testing"
+
+	"github.com/OJ/gobuster/v3/libgobuster"
+)
+
+func TestAdditionalWordsLen(t *testing.T) {
+	t.Parallel()
+
+	tt := []struct {
+		testName   string
+		extensions map[string]bool
+	}{
+		{"No extensions", map[string]bool{}},
+		{"Some extensions", map[string]bool{"htm": true, "html": true, "php": true}},
+	}
+
+	globalOpts := libgobuster.Options{}
+	for _, x := range tt {
+		opts := OptionsDir{}
+		opts.ExtensionsParsed.Set = x.extensions
+
+		d, _ := New(&globalOpts, &opts, nil)
+
+		calculatedLen := d.AdditionalWordsLen()
+		wordsLen := len(d.AdditionalWords("dummy"))
+
+		if calculatedLen != wordsLen {
+			t.Fatalf("Mismatched additional words length: %d got %d generated words %v", calculatedLen, wordsLen, d.AdditionalWords("dummy"))
+		}
+	}
+}