Merge pull request #818 from projectdiscovery/dev

Nuclei v2.4.0 Release
projectdiscovery · Jul 9, 2021 · 76fe4c6 · 76fe4c6
2 parents 84ff74c + 58fc6b8
commit 76fe4c6
Show file tree

Hide file tree

Showing 47 changed files with 1,613 additions and 785 deletions.
diff --git a/README.md b/README.md
@@ -45,22 +45,20 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 # Install Nuclei
 
 ```sh
-▶ GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
+GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 ```
 
-**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started.html).**
+**More installation [methods can be found here](https://nuclei.projectdiscovery.io/nuclei/get-started/).**
 
 <table>
 <tr>
 <td>  
 
-### Download Templates
+### Nuclei Templates
 
-You can download and update the nuclei templates using <ins>*update-templates*</ins> flag of nuclei that downloads all the available **nuclei-templates** from [Github project](https://github.com/projectdiscovery/nuclei-templates), a community curated list of templates that are ready to use.
+Nuclei has had built-in support for automatic update/download templates since version [v2.4.0](https://github.com/projectdiscovery/nuclei/releases/tag/v2.4.0). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
 
-`▶ nuclei -update-templates`
-
-Nuclei is designed to used with custom templates according to the target and workflow, you can write your own checks for your specific workflow and needs, please refer to nuclei [templating guide](https://nuclei.projectdiscovery.io/templating-guide/) to write your own custom templates.
+You may still use the `update-templates` flag to update the nuclei templates at any time; automatic updates happen every 24 hours. You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/).
 
 </td>
 </tr>
@@ -82,74 +80,94 @@ Usage:
   nuclei [flags]
 
 Flags:
-   -H, -header value                      Custom Header.
-   -biid, -burp-collaborator-biid string  Burp Collaborator BIID
-   -bs, -bulk-size int                    Maximum Number of hosts analyzed in parallel per template (default 25)
-   -c, -concurrency int                   Maximum Number of templates executed in parallel (default 10)
-   -config string                         Nuclei configuration file
-   -debug                                 Debugging request and responses
-   -debug-req                             Debugging request
-   -debug-resp                            Debugging response
-   -et, -exclude value                    Templates to exclude, supports single and multiple templates using directory.
-   -etags, -exclude-tags value            Exclude templates with the provided tags
-   -headless                              Enable headless browser based templates support
-   -impact, -severity value               Templates to run based on severity, supports single and multiple severity.
-   -irr, -include-rr                      Write requests/responses for matches in JSON output
-   -interactions-cache-size int           Number of requests to keep in interactions cache (default 5000)
-   -interactions-cooldown-period int      Extra time for interaction polling before exiting (default 5)
-   -interactions-eviction int             Number of seconds to wait before evicting requests from cache (default 60)
-   -interactions-poll-duration int        Number of seconds before each interaction poll request (default 5)
-   -interactsh-url string                 Self Hosted Interactsh Server URL (default https://interact.sh)
-   -json                                  Write json output to files
-   -l, -list string                       List of URLs to run templates on
-   -me, -markdown-export string           Directory to export results in markdown format
-   -metrics                               Expose nuclei metrics on a port
-   -metrics-port int                      Port to expose nuclei metrics on (default 9092)
-   -nc, -no-color                         Disable colors in output
-   -nt, -new-templates                    Only run newly added templates
-   -nm, -no-meta                          Don't display metadata for the matches
-   -no-interactsh                         Do not use interactsh server for blind interaction polling
-   -o, -output string                     File to write output to (optional)
-   -page-timeout int                      Seconds to wait for each page in headless (default 20)
-   -passive                               Enable Passive HTTP response processing mode
-   -project                               Use a project folder to avoid sending same request multiple times
-   -project-path string                   Use a user defined project folder, temporary folder is used if not specified but enabled
-   -proxy-socks-url string                URL of the proxy socks server
-   -proxy-url string                      URL of the proxy server
-   -r, -resolvers string                  File containing resolver list for nuclei
-   -rl, -rate-limit int                   Maximum requests to send per second (default 150)
-   -rc, -report-config string             Nuclei Reporting Module configuration file
-   -rdb, -report-db string                Local Nuclei Reporting Database (Always use this to persistent report data)
-   -retries int                           Number of times to retry a failed request (default 1)
-   -se, -sarif-export string              File to export results in sarif format
-   -show-browser                          Show the browser on the screen
-   -si, -stats-interval int               Number of seconds between each stats line (default 5)
-   -silent                                Show only results in output
-   -spm, -stop-at-first-path              Stop processing http requests at first match (this may break template/workflow logic)
-   -stats                                 Display stats of the running scan
-   -system-resolvers                      Use system dns resolving as error fallback
-   -t, -templates value                   Templates to run, supports single and multiple templates using directory.
-   -tags value                            Tags to execute templates for
-   -u, -target string                     URL to scan with nuclei
-   -tv, -templates-version                Shows the installed nuclei-templates version
-   -timeout int                           Time to wait in seconds before timeout (default 5)
-   -tl                                    List available templates
-   -trace-log string                      File to write sent requests trace log
-   -ud, -update-directory string          Directory storing nuclei-templates (default /Users/geekboy/nuclei-templates)
-   -ut, -update-templates                 Download / updates nuclei community templates
-   -v, -verbose                           Show verbose output
-   -version                               Show version of nuclei
-   -w, -workflows value                   Workflows to run for nuclei
+   -H, -header value                  Custom Header.
+   -author value                      Templates to run based on author
+   -bs, -bulk-size int                Maximum Number of hosts analyzed in parallel per template (default 25)
+   -c, -concurrency int               Maximum Number of templates executed in parallel (default 10)
+   -config string                     Nuclei configuration file
+   -debug                             Debugging request and responses
+   -debug-req                         Debugging request
+   -debug-resp                        Debugging response
+   -et, -exclude value                Templates to exclude, supports single and multiple templates using directory.
+   -etags, -exclude-tags value        Exclude templates with the provided tags
+   -headless                          Enable headless browser based templates support
+   -impact, -severity value           Templates to run based on severity
+   -irr, -include-rr                  Write requests/responses for matches in JSON output
+   -include-tags value                Tags to force run even if they are in denylist
+   -include-templates value           Templates to force run even if they are in denylist
+   -interactions-cache-size int       Number of requests to keep in interactions cache (default 5000)
+   -interactions-cooldown-period int  Extra time for interaction polling before exiting (default 5)
+   -interactions-eviction int         Number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int    Number of seconds before each interaction poll request (default 5)
+   -interactsh-url string             Self Hosted Interactsh Server URL (default https://interact.sh)
+   -json                              Write json output to files
+   -l, -list string                   List of URLs to run templates on
+   -me, -markdown-export string       Directory to export results in markdown format
+   -metrics                           Expose nuclei metrics on a port
+   -metrics-port int                  Port to expose nuclei metrics on (default 9092)
+   -nc, -no-color                     Disable colors in output
+   -nt, -new-templates                Only run newly added templates
+   -nm, -no-meta                      Don't display metadata for the matches
+   -no-interactsh                     Do not use interactsh server for blind interaction polling
+   -o, -output string                 File to write output to (optional)
+   -page-timeout int                  Seconds to wait for each page in headless (default 20)
+   -passive                           Enable Passive HTTP response processing mode
+   -project                           Use a project folder to avoid sending same request multiple times
+   -project-path string               Use a user defined project folder, temporary folder is used if not specified but enabled
+   -proxy-socks-url string            URL of the proxy socks server
+   -proxy-url string                  URL of the proxy server
+   -r, -resolvers string              File containing resolver list for nuclei
+   -rl, -rate-limit int               Maximum requests to send per second (default 150)
+   -rc, -report-config string         Nuclei Reporting Module configuration file
+   -rdb, -report-db string            Local Nuclei Reporting Database (Always use this to persistent report data)
+   -retries int                       Number of times to retry a failed request (default 1)
+   -se, -sarif-export string          File to export results in sarif format
+   -show-browser                      Show the browser on the screen
+   -si, -stats-interval int           Number of seconds between each stats line (default 5)
+   -silent                            Show only results in output
+   -spm, -stop-at-first-path          Stop processing http requests at first match (this may break template/workflow logic)
+   -stats                             Display stats of the running scan
+   -stats-json                        Write stats output in JSON format
+   -system-resolvers                  Use system dns resolving as error fallback
+   -t, -templates value               Templates to run, supports single and multiple templates using directory.
+   -tags value                        Tags to execute templates for
+   -u, -target string                 URL to scan with nuclei
+   -tv, -templates-version            Shows the installed nuclei-templates version
+   -timeout int                       Time to wait in seconds before timeout (default 5)
+   -tl                                List available templates
+   -trace-log string                  File to write sent requests trace log
+   -ud, -update-directory string      Directory storing nuclei-templates (default /Users/geekboy/nuclei-templates)
+   -ut, -update-templates             Download / updates nuclei community templates
+   -v, -verbose                       Show verbose output
+   -validate                          Validate the passed templates to nuclei
+   -version                           Show version of nuclei
+   -vv                                Display Extra Verbose Information
+   -w, -workflows value               Workflows to run for nuclei
 ```
 
 </details>
 
 ### Running Nuclei
 
-Scanning for CVEs on given list of URLs.
+Scanning target domain with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
+
+```sh
+nuclei -target https://example.com
+```
+
+Scanning target URLs with [community-curated](https://github.com/projectdiscovery/nuclei-templates) nuclei templates.
 
 ```sh
-▶ nuclei -l target_urls.txt -t cves/
+nuclei -list urls.txt
+```
+
+Example of `urls.txt`:
+
+```yaml
+http://example.com
+http://app.example.com
+http://test.example.com
+http://uat.example.com
 ```
 
 **More detailed examples of running nuclei can be found [here](https://nuclei.projectdiscovery.io/nuclei/get-started/#running-nuclei).**

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
@@ -24,7 +24,9 @@ func main() {
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
-	nucleiRunner.RunEnumeration()
+	if err := nucleiRunner.RunEnumeration(); err != nil {
+		gologger.Fatal().Msgf("Could not run nuclei: %s\n", err)
+	}
 	nucleiRunner.Close()
 }
 
@@ -41,11 +43,14 @@ based on templates offering massive extensibility and ease of use.`)
 	set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei")
 	set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
 	set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
-	set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
-	set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
+	set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
+	set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity")
+	set.StringSliceVar(&options.Author, "author", []string{}, "Templates to run based on author")
+	set.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "Templates to force run even if they are in denylist")
+	set.StringSliceVar(&options.IncludeTags, "include-tags", []string{}, "Tags to force run even if they are in denylist")
 	set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
 	set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
-	set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
+	set.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the proxy server")
 	set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server")
 	set.BoolVar(&options.Silent, "silent", false, "Show only results in output")
 	set.BoolVar(&options.Version, "version", false, "Show version of nuclei")
@@ -81,6 +86,7 @@ based on templates offering massive extensibility and ease of use.`)
 	set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support")
 	set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen")
 	set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line")
+	set.BoolVar(&options.StatsJSON, "stats-json", false, "Write stats output in JSON format")
 	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
 	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
 	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
@@ -92,6 +98,8 @@ based on templates offering massive extensibility and ease of use.`)
 	set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache")
 	set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request")
 	set.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "Extra time for interaction polling before exiting")
+	set.BoolVar(&options.VerboseVerbose, "vv", false, "Display Extra Verbose Information")
+	set.BoolVar(&options.Validate, "validate", false, "Validate the passed templates to nuclei")
 	_ = set.Parse()
 
 	if cfgFile != "" {

diff --git a/v2/go.mod b/v2/go.mod
@@ -21,12 +21,10 @@ require (
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mattn/go-runewidth v0.0.10 // indirect
 	github.com/miekg/dns v1.1.38
-	github.com/mitchellh/go-ps v1.0.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/owenrumney/go-sarif v1.0.4
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/collaborator v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.8
 	github.com/projectdiscovery/goflags v0.0.4
 	github.com/projectdiscovery/gologger v1.1.4
@@ -35,10 +33,12 @@ require (
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.10
 	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
+	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rs/xid v1.2.1
 	github.com/segmentio/ksuid v1.0.3
+	github.com/shirou/gopsutil/v3 v3.21.5
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.3.1
 	github.com/stretchr/testify v1.7.0