diff --git a/README.md b/README.md index 319f478c1b..f898e326a4 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ TEMPLATES: -nss, -no-strict-syntax disable strict syntax check on templates -td, -template-display displays the templates content -tl list all available templates + -tgl list all available tags -sign signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable -code enable loading code protocol-based templates -dut, -disable-unsigned-templates disable running unsigned templates or templates with mismatched signature diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 7685141389..180efcfc6e 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -37,6 +37,7 @@ var ( "dns": dnsTestCases, "workflow": workflowTestcases, "loader": loaderTestcases, + "profile-loader": profileLoaderTestcases, "websocket": websocketTestCases, "headless": headlessTestcases, "whois": whoisTestCases, diff --git a/cmd/integration-test/profile-loader.go b/cmd/integration-test/profile-loader.go new file mode 100644 index 0000000000..dafc15aa21 --- /dev/null +++ b/cmd/integration-test/profile-loader.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + + "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + errorutil "github.com/projectdiscovery/utils/errors" +) + +var profileLoaderTestcases = []TestCaseInfo{ + {Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}}, + {Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}}, + {Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}}, +} + +type profileLoaderByRelFile struct{} + +func (h *profileLoaderByRelFile) Execute(testName string) error { + results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud.yml") + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to load template with id") + } + if len(results) <= 10 { + return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results)) + } + return nil +} + +type profileLoaderById struct{} + +func (h *profileLoaderById) Execute(testName string) error { + results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud") + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to load template with id") + } + if len(results) <= 10 { + return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results)) + } + return nil +} + +// this profile with load kevs +type customProfileLoader struct{} + +func (h *customProfileLoader) Execute(filepath string) error { + results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", filepath) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to load template with id") + } + if len(results) < 1 { + return fmt.Errorf("incorrect result: expected more results than %d, got %v", 1, len(results)) + } + return nil +} diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 242a902f55..a1123b9ffc 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -42,9 +42,10 @@ import ( ) var ( - cfgFile string - memProfile string // optional profile file path - options = &types.Options{} + cfgFile string + templateProfile string + memProfile string // optional profile file path + options = &types.Options{} ) func main() { @@ -199,7 +200,7 @@ on extensive configurability, massive extensibility and ease of use.`) */ flagSet.CreateGroup("input", "Target", - flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions), flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"), flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"), @@ -225,6 +226,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"), flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), + flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"), flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"), flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"), flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"), @@ -270,6 +272,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("configs", "Configurations", flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"), + flagSet.StringVarP(&templateProfile, "profile", "tp", "", "template profile config file to run"), + flagSet.BoolVarP(&options.ListTemplateProfiles, "profile-list", "tpl", false, "list community template profiles"), flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"), flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"), flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"), @@ -409,7 +413,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("cloud", "Cloud", flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"), - flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"), + flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"), + flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"), ) flagSet.CreateGroup("Authentication", "Authentication", @@ -497,6 +502,34 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory) } + defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles") + if templateProfile != "" { + if filepath.Ext(templateProfile) == "" { + if tp := findProfilePathById(templateProfile, defaultProfilesPath); tp != "" { + templateProfile = tp + } else { + gologger.Fatal().Msgf("'%s' is not a profile-id or profile path", templateProfile) + } + } + if !filepath.IsAbs(templateProfile) { + if filepath.Dir(templateProfile) == "profiles" { + defaultProfilesPath = filepath.Join(config.DefaultConfig.GetTemplateDir()) + } + currentDir, err := os.Getwd() + if err == nil && fileutil.FileExists(filepath.Join(currentDir, templateProfile)) { + templateProfile = filepath.Join(currentDir, templateProfile) + } else { + templateProfile = filepath.Join(defaultProfilesPath, templateProfile) + } + } + if !fileutil.FileExists(templateProfile) { + gologger.Fatal().Msgf("given template profile file '%s' does not exist", templateProfile) + } + if err := flagSet.MergeConfigFile(templateProfile); err != nil { + gologger.Fatal().Msgf("Could not read template profile: %s\n", err) + } + } + if len(options.SecretsFile) > 0 { for _, secretFile := range options.SecretsFile { if !fileutil.FileExists(secretFile) { @@ -622,6 +655,27 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin os.Exit(0) } +func findProfilePathById(profileId, templatesDir string) string { + var profilePath string + err := filepath.WalkDir(templatesDir, func(iterItem string, d fs.DirEntry, err error) error { + ext := filepath.Ext(iterItem) + isYaml := ext == extensions.YAML || ext == extensions.YML + if err != nil || d.IsDir() || !isYaml { + // skip non yaml files + return nil + } + if strings.TrimSuffix(filepath.Base(iterItem), ext) == profileId { + profilePath = iterItem + return fmt.Errorf("FOUND") + } + return nil + }) + if err != nil && err.Error() != "FOUND" { + gologger.Error().Msgf("%s\n", err) + } + return profilePath +} + func init() { // print stacktrace of errors in debug mode if strings.EqualFold(os.Getenv("DEBUG"), "true") { diff --git a/go.mod b/go.mod index 100a05d0be..1002e41efd 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.69 + github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.47 - github.com/projectdiscovery/retryabledns v1.0.58 - github.com/projectdiscovery/retryablehttp-go v1.0.58 + github.com/projectdiscovery/retryabledns v1.0.59 + github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -93,9 +93,9 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 - github.com/projectdiscovery/useragent v0.0.48 + github.com/projectdiscovery/useragent v0.0.49 github.com/projectdiscovery/utils v0.0.92 - github.com/projectdiscovery/wappalyzergo v0.0.120 + github.com/projectdiscovery/wappalyzergo v0.0.122 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 0363c74eae..dbc2f68aeb 100644 --- a/go.sum +++ b/go.sum @@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= -github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= +github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= +github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -875,10 +875,10 @@ github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5 github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= -github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4= -github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0= +github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= +github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= +github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= +github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -887,12 +887,12 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= -github.com/projectdiscovery/useragent v0.0.48 h1:ITygElwcY9FlOt0F65kcW/oAALNr1nQOtO3kR9lRzaY= -github.com/projectdiscovery/useragent v0.0.48/go.mod h1:ahQMoWlVNFVQxjHOKqOPHJJQ7Ovz1zJZuJbBsAwIcOw= +github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg= +github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0= github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= -github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= -github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= +github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c= +github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= diff --git a/integration_tests/profile-loader/basic.yml b/integration_tests/profile-loader/basic.yml new file mode 100644 index 0000000000..6b53297003 --- /dev/null +++ b/integration_tests/profile-loader/basic.yml @@ -0,0 +1,2 @@ +tags: + - kev \ No newline at end of file diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index bd0a7d4712..a76d4ea31a 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/url" + "regexp" "sync/atomic" "time" @@ -27,9 +28,13 @@ const ( appendEndpoint = "/v1/scans/%s/import" flushTimer = time.Duration(1) * time.Minute MaxChunkSize = 1024 * 1024 * 4 // 4 MB + xidRe = `^[a-z0-9]{20}$` ) -var _ output.Writer = &UploadWriter{} +var ( + xidRegex = regexp.MustCompile(xidRe) + _ output.Writer = &UploadWriter{} +) // UploadWriter is a writer that uploads its output to pdcp // server to enable web dashboard and more @@ -41,6 +46,7 @@ type UploadWriter struct { cancel context.CancelFunc done chan struct{} scanID string + scanName string counter atomic.Int32 } @@ -86,8 +92,17 @@ func NewUploadWriter(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*Upl } // SetScanID sets the scan id for the upload writer -func (u *UploadWriter) SetScanID(id string) { +func (u *UploadWriter) SetScanID(id string) error { + if !xidRegex.MatchString(id) { + return fmt.Errorf("invalid scan id provided") + } u.scanID = id + return nil +} + +// SetScanName sets the scan name for the upload writer +func (u *UploadWriter) SetScanName(name string) { + u.scanName = name } func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) { @@ -220,7 +235,13 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request") } // add pdtm meta params - req.URL.RawQuery = updateutils.GetpdtmParams(config.Version) + req.URL.Params.Merge(updateutils.GetpdtmParams(config.Version)) + // if it is upload endpoint also include name if it exists + if u.scanName != "" && req.URL.Path == uploadEndpoint { + req.URL.Params.Add("name", u.scanName) + } + req.URL.Update() + req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", "application/json") diff --git a/internal/runner/options.go b/internal/runner/options.go index bc89a45bdd..cad5202f8a 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -3,6 +3,7 @@ package runner import ( "bufio" "fmt" + "io/fs" "os" "path/filepath" "strconv" @@ -25,6 +26,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif" + "github.com/projectdiscovery/nuclei/v3/pkg/templates/extensions" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" fileutil "github.com/projectdiscovery/utils/file" @@ -74,6 +76,31 @@ func ParseOptions(options *types.Options) { } os.Exit(0) } + + defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles") + if options.ListTemplateProfiles { + gologger.Print().Msgf( + "\nListing available %v nuclei template profiles for %v", + config.DefaultConfig.TemplateVersion, + config.DefaultConfig.TemplatesDirectory, + ) + templatesRootDir := config.DefaultConfig.GetTemplateDir() + err := filepath.WalkDir(defaultProfilesPath, func(iterItem string, d fs.DirEntry, err error) error { + ext := filepath.Ext(iterItem) + isYaml := ext == extensions.YAML || ext == extensions.YML + if err != nil || d.IsDir() || !isYaml { + return nil + } + if profileRelPath, err := filepath.Rel(templatesRootDir, iterItem); err == nil { + gologger.Print().Msgf("%s (%s)\n", profileRelPath, strings.TrimSuffix(filepath.Base(iterItem), ext)) + } + return nil + }) + if err != nil { + gologger.Error().Msgf("%s\n", err) + } + os.Exit(0) + } if options.StoreResponseDir != DefaultDumpTrafficOutputFolder && !options.StoreResponse { gologger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n") options.StoreResponse = true diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 43f16ff10b..a0c46f6d03 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -414,7 +414,11 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer { return writer } if r.options.ScanID != "" { - uploadWriter.SetScanID(r.options.ScanID) + // ignore and use empty scan id if invalid + _ = uploadWriter.SetScanID(r.options.ScanID) + } + if r.options.ScanName != "" { + uploadWriter.SetScanName(r.options.ScanName) } return output.NewMultiWriter(writer, uploadWriter) } @@ -510,6 +514,23 @@ func (r *Runner) RunEnumeration() error { return errors.Wrap(err, "Could not create loader.") } + // list all templates or tags as specified by user. + // This uses a separate parser to reduce time taken as + // normally nuclei does a lot of compilation and stuff + // for templates, which we don't want for these simp + if r.options.TemplateList || r.options.TemplateDisplay || r.options.TagList { + if err := store.LoadTemplatesOnlyMetadata(); err != nil { + return err + } + + if r.options.TagList { + r.listAvailableStoreTags(store) + } else { + r.listAvailableStoreTemplates(store) + } + os.Exit(0) + } + if r.options.Validate { if err := store.ValidateTemplates(); err != nil { return err @@ -540,12 +561,6 @@ func (r *Runner) RunEnumeration() error { _ = r.inputProvider.SetWithExclusions(host) } } - // list all templates - if r.options.TemplateList || r.options.TemplateDisplay { - r.listAvailableStoreTemplates(store) - os.Exit(0) - } - // display execution info like version , templates used etc r.displayExecutionInfo(store) diff --git a/internal/runner/templates.go b/internal/runner/templates.go index ef85f6cb99..aaa08dd663 100644 --- a/internal/runner/templates.go +++ b/internal/runner/templates.go @@ -3,9 +3,11 @@ package runner import ( "bytes" "path/filepath" + "sort" "strings" "github.com/alecthomas/chroma/quick" + jsoniter "github.com/json-iterator/go" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" @@ -71,6 +73,40 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { } } +func (r *Runner) listAvailableStoreTags(store *loader.Store) { + gologger.Print().Msgf( + "\nListing available %v nuclei tags for %v", + config.DefaultConfig.TemplateVersion, + config.DefaultConfig.TemplatesDirectory, + ) + tagsMap := make(map[string]int) + for _, tpl := range store.Templates() { + for _, tag := range tpl.Info.Tags.ToSlice() { + tagsMap[tag]++ + } + } + type kv struct { + Key string `json:"tag"` + Value int `json:"count"` + } + var tagsList []kv + for k, v := range tagsMap { + tagsList = append(tagsList, kv{k, v}) + } + sort.Slice(tagsList, func(i, j int) bool { + return tagsList[i].Value > tagsList[j].Value + }) + + for _, tag := range tagsList { + if r.options.JSONL { + marshalled, _ := jsoniter.Marshal(tag) + gologger.Silent().Msgf("%s\n", string(marshalled)) + } else { + gologger.Silent().Msgf("%s (%d)\n", tag.Key, tag.Value) + } + } +} + func (r *Runner) highlightTemplate(body *[]byte) ([]byte, error) { var buf bytes.Buffer // YAML lexer, true color terminal formatter and monokai style diff --git a/lib/multi.go b/lib/multi.go index 726b59c5d9..5b2c7d6776 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -111,11 +111,11 @@ func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *outpu e.eng.resultCallbacks = []func(*output.ResultEvent){callback} } -// ExecuteNucleiWithOpts executes templates on targets and calls callback on each result(only if results are found) +// ExecuteNucleiWithOptsCtx executes templates on targets and calls callback on each result(only if results are found) // This method can be called concurrently and it will use some global resources but can be runned parallelly // by invoking this method with different options and targets // Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options -func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error { +func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, targets []string, opts ...NucleiSDKOptions) error { baseOpts := *e.eng.opts tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe} for _, option := range opts { @@ -163,6 +163,12 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts .. return nil } +// ExecuteNucleiWithOpts is same as ExecuteNucleiWithOptsCtx but with default context +// This is a placeholder and will be deprecated in future major release +func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error { + return e.ExecuteNucleiWithOptsCtx(context.Background(), targets, opts...) +} + // Close all resources used by nuclei engine func (e *ThreadSafeNucleiEngine) Close() { e.eng.Close() diff --git a/lib/sdk.go b/lib/sdk.go index 7959d4c1d5..63925f47ca 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -219,8 +219,9 @@ func (e *NucleiEngine) Close() { } } -// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) -func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error { +// ExecuteCallbackWithCtx executes templates on targets and calls callback on each result(only if results are found) +// enable matcher-status option if you expect this callback to be called for all results regardless if it matched or not +func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...func(event *output.ResultEvent)) error { if !e.templatesLoaded { _ = e.LoadAllTemplates() } @@ -244,10 +245,18 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result return nil } +// ExecuteWithCallback is same as ExecuteCallbackWithCtx but with default context +// Note this is deprecated and will be removed in future major release +func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error { + return e.ExecuteCallbackWithCtx(context.Background(), callback...) +} + +// Options return nuclei Type Options func (e *NucleiEngine) Options() *types.Options { return e.opts } +// Engine returns core Executer of nuclei func (e *NucleiEngine) Engine() *core.Engine { return e.engine } diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index aec23c93cc..f66447eae5 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.2.7-dev` + Version = `v3.2.8-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 711008f2f0..c719f71a5a 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -255,6 +255,46 @@ func init() { templateIDPathMap = make(map[string]string) } +// LoadTemplatesOnlyMetadata loads only the metadata of the templates +func (store *Store) LoadTemplatesOnlyMetadata() error { + templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates) + store.logErroredTemplates(errs) + + filteredTemplatePaths := store.pathFilter.Match(templatePaths) + + validPaths := make(map[string]struct{}) + for templatePath := range filteredTemplatePaths { + loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog) + if loaded || store.pathFilter.MatchIncluded(templatePath) { + validPaths[templatePath] = struct{}{} + } + if err != nil { + if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { + stats.Increment(templates.TemplatesExcludedStats) + if config.DefaultConfig.LogAllEvents { + gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) + } + continue + } + gologger.Warning().Msg(err.Error()) + } + } + parserItem, ok := store.config.ExecutorOptions.Parser.(*templates.Parser) + if !ok { + return errors.New("invalid parser") + } + templatesCache := parserItem.Cache() + + for templatePath := range validPaths { + template, _, _ := templatesCache.Has(templatePath) + if template != nil { + template.Path = templatePath + store.templates = append(store.templates, template) + } + } + return nil +} + // ValidateTemplates takes a list of templates and validates them // erroring out on discovering any faulty templates. func (store *Store) ValidateTemplates() error { diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 3cf6a2ade1..86302bb713 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -162,7 +162,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t // wp is workpool that contains different waitgroups for // headless and non-headless templates // global waitgroup should not be used here - wp := e.WorkPool() + wp := e.GetWorkPool() for _, tpl := range alltemplates { select { diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index 23b3e6e976..bd1414638e 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -57,6 +57,8 @@ type GeneratedRequest struct { DynamicValues map[string]interface{} // Component is the component for the request Component component.Component + // Parameter being fuzzed + Parameter string } // Execute executes a fuzzing rule accepting a callback on which @@ -223,7 +225,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo if err != nil { return err } - if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); gotErr != nil { + if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil { return gotErr } } diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index 2796a7dc96..4c01135f66 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -68,7 +68,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil { return qerr } // fmt.Printf("executed with value: %s\n", evaluated) @@ -90,7 +90,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS if err != nil { return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); qerr != nil { err = qerr return err } @@ -125,7 +125,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil { return err } @@ -144,12 +144,13 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu } // execWithInput executes a rule with input via callback -func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component) error { +func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter string) error { request := GeneratedRequest{ Request: httpReq, InteractURLs: interactURLs, DynamicValues: input.Values, Component: component, + Parameter: parameter, } if !input.Callback(request) { return types.ErrNoMoreRequests diff --git a/pkg/model/types/severity/severities.go b/pkg/model/types/severity/severities.go index a4eb136e26..193a9e45e5 100644 --- a/pkg/model/types/severity/severities.go +++ b/pkg/model/types/severity/severities.go @@ -26,6 +26,14 @@ func (severities *Severities) Set(values string) error { return nil } +func (severities Severities) MarshalYAML() (interface{}, error) { + var stringSeverities = make([]string, 0, len(severities)) + for _, severity := range severities { + stringSeverities = append(stringSeverities, severity.String()) + } + return stringSeverities, nil +} + func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) error { var stringSliceValue stringslice.StringSlice if err := unmarshal(&stringSliceValue); err != nil { diff --git a/pkg/model/types/severity/severity_test.go b/pkg/model/types/severity/severity_test.go index e52ed42f4f..10d2050de3 100644 --- a/pkg/model/types/severity/severity_test.go +++ b/pkg/model/types/severity/severity_test.go @@ -74,3 +74,12 @@ func TestMarshalJSON(t *testing.T) { } require.Equal(t, "[\"low\",\"medium\"]", string(data), "could not marshal json") } + +func TestSeveritiesMarshalYaml(t *testing.T) { + unmarshalled := Severities{Low, Medium} + marshalled, err := yaml.Marshal(unmarshalled) + if err != nil { + panic(err) + } + require.Equal(t, "- low\n- medium\n", string(marshalled), "could not marshal yaml") +} diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 2d6310df3f..9d0f03efaf 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -106,5 +106,21 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { } builder.WriteString("]") } + + // If it is a fuzzing output, enrich with additional + // metadata for the match. + if output.IsFuzzingResult { + if output.FuzzingParameter != "" { + builder.WriteString(" [") + builder.WriteString(output.FuzzingPosition) + builder.WriteRune(':') + builder.WriteString(w.aurora.BrightMagenta(output.FuzzingParameter).String()) + builder.WriteString("]") + } + + builder.WriteString(" [") + builder.WriteString(output.FuzzingMethod) + builder.WriteString("]") + } return builder.Bytes() } diff --git a/pkg/output/output.go b/pkg/output/output.go index 4a85c32480..044d164f1f 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -175,6 +175,14 @@ type ResultEvent struct { // must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true ReqURLPattern string `json:"req_url_pattern,omitempty"` + // Fields related to HTTP Fuzzing functionality of nuclei. + // The output contains additional fields when the result is + // for a fuzzing template. + IsFuzzingResult bool `json:"is_fuzzing_result,omitempty"` + FuzzingMethod string `json:"fuzzing_method,omitempty"` + FuzzingParameter string `json:"fuzzing_parameter,omitempty"` + FuzzingPosition string `json:"fuzzing_position,omitempty"` + FileToIndexPosition map[string]int `json:"-"` Error string `json:"error,omitempty"` } diff --git a/pkg/protocols/common/uncover/uncover.go b/pkg/protocols/common/uncover/uncover.go index a3f221334a..e8a0d41204 100644 --- a/pkg/protocols/common/uncover/uncover.go +++ b/pkg/protocols/common/uncover/uncover.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/uncover" "github.com/projectdiscovery/uncover/sources" mapsutil "github.com/projectdiscovery/utils/maps" + sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -84,9 +85,21 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T if queriesMap[engine] == nil { queriesMap[engine] = []string{} } - queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v)) + switch v := v.(type) { + case []interface{}: + qs := queriesMap[engine] + for _, vv := range v { + qs = append(qs, fmt.Sprint(vv)) + } + queriesMap[engine] = qs + default: + queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v)) + } } } + for engine, queries := range queriesMap { + queriesMap[engine] = sliceutil.Dedupe(queries) + } keys := mapsutil.GetKeys(queriesMap) gologger.Info().Msgf("Running uncover queries from template against: %s", strings.Join(keys, ",")) result := make(chan string, runtime.NumCPU()) @@ -97,8 +110,11 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T // TODO: add support for map[engine]queries in uncover // Note below implementation is intentionally sequential to avoid burning all the API keys counter := 0 - + outerLoop: for eng, queries := range queriesMap { + if opts.Limit > 0 && counter >= opts.Limit { + break + } // create new uncover options for each engine uncoverOpts := &uncover.Options{ Agents: []string{eng}, @@ -114,18 +130,20 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T gologger.Error().Msgf("Could not get targets using %v engine from uncover: %s", eng, err) return } + + innerLoop: for { select { case <-ctx.Done(): return case res, ok := <-ch: if !ok { - return + continue outerLoop } result <- res counter++ if opts.Limit > 0 && counter >= opts.Limit { - return + break innerLoop } } } diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index 42065576ad..bc5f9caa9b 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -170,6 +170,13 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, } var gotMatches bool requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) { + for _, result := range event.Results { + result.IsFuzzingResult = true + result.FuzzingMethod = gr.Request.Method + result.FuzzingParameter = gr.Parameter + result.FuzzingPosition = gr.Component.Name() + } + if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index c9fe0f4bab..08f32e9313 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -276,6 +276,7 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error { if tracker.ShouldFilter(event) { continue } + trackerName := tracker.Name() stats, statsOk := c.stats[trackerName] diff --git a/pkg/reporting/trackers/gitea/gitea.go b/pkg/reporting/trackers/gitea/gitea.go index 70fbe7fe51..acb82a1de2 100644 --- a/pkg/reporting/trackers/gitea/gitea.go +++ b/pkg/reporting/trackers/gitea/gitea.go @@ -140,7 +140,7 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error { // ShouldFilter determines if an issue should be logged to this tracker func (i *Integration) ShouldFilter(event *output.ResultEvent) bool { if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) { - return true + return false } if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) { diff --git a/pkg/reporting/trackers/github/github.go b/pkg/reporting/trackers/github/github.go index 102692e6ce..44c06102ca 100644 --- a/pkg/reporting/trackers/github/github.go +++ b/pkg/reporting/trackers/github/github.go @@ -176,7 +176,7 @@ func (i *Integration) Name() string { // ShouldFilter determines if an issue should be logged to this tracker func (i *Integration) ShouldFilter(event *output.ResultEvent) bool { if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) { - return true + return false } if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) { diff --git a/pkg/reporting/trackers/gitlab/gitlab.go b/pkg/reporting/trackers/gitlab/gitlab.go index dc5bef0d6e..769165ad48 100644 --- a/pkg/reporting/trackers/gitlab/gitlab.go +++ b/pkg/reporting/trackers/gitlab/gitlab.go @@ -165,7 +165,7 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error { // ShouldFilter determines if an issue should be logged to this tracker func (i *Integration) ShouldFilter(event *output.ResultEvent) bool { if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) { - return true + return false } if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) { diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go index 97ae078312..0b8f27fc5b 100644 --- a/pkg/reporting/trackers/jira/jira.go +++ b/pkg/reporting/trackers/jira/jira.go @@ -315,7 +315,7 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (jira.Issue, // ShouldFilter determines if an issue should be logged to this tracker func (i *Integration) ShouldFilter(event *output.ResultEvent) bool { if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) { - return true + return false } if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) { diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index b821fcc675..39d6a6ea0f 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" @@ -341,6 +342,18 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e return nil, errors.New("no template author field provided") } + // use default unknown severity + if len(template.Workflows) == 0 { + if template.Info.SeverityHolder.Severity == severity.Undefined { + // set unknown severity with counter and forced warning + template.Info.SeverityHolder.Severity = severity.Unknown + if options.Options.Validate { + // when validating return error + return nil, errors.New("no template severity field provided") + } + } + } + // Setting up variables regarding template metadata options.TemplateID = template.ID options.TemplateInfo = template.Info diff --git a/pkg/templates/parser.go b/pkg/templates/parser.go index 9d515c0657..f76352e76c 100644 --- a/pkg/templates/parser.go +++ b/pkg/templates/parser.go @@ -33,6 +33,11 @@ func NewParser() *Parser { return p } +// Cache returns the parsed templates cache +func (p *Parser) Cache() *Cache { + return p.parsedTemplatesCache +} + // LoadTemplate returns true if the template is valid and matches the filtering criteria. func (p *Parser) LoadTemplate(templatePath string, t any, extraTags []string, catalog catalog.Catalog) (bool, error) { tagFilter, ok := t.(*TagFilter) diff --git a/pkg/types/types.go b/pkg/types/types.go index d7dd41f22d..0c31769ee1 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -225,6 +225,8 @@ type Options struct { TemplateDisplay bool // TemplateList lists available templates TemplateList bool + // TemplateList lists available tags + TagList bool // HangMonitor enables nuclei hang monitoring HangMonitor bool // Stdin specifies whether stdin input was given to the process @@ -375,6 +377,8 @@ type Options struct { EnableCloudUpload bool // ScanID is the scan ID to use for cloud upload ScanID string + // ScanName is the name of the scan to be uploaded + ScanName string // JsConcurrency is the number of concurrent js routines to run JsConcurrency int // SecretsFile is file containing secrets for nuclei @@ -393,6 +397,8 @@ type Options struct { DAST bool // HttpApiEndpoint is the experimental http api endpoint HttpApiEndpoint string + // ListTemplateProfiles lists all available template profiles + ListTemplateProfiles bool } // ShouldLoadResume resume file