From 5c75eaec5d5d51573dba45423319fa11ea724840 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 24 Aug 2023 02:01:43 +0530 Subject: [PATCH 01/10] new sdk progress --- v2/config.go | 212 ++++++++++++++++++++++++++++++++++++++++++++++ v2/config_test.go | 8 ++ v2/sdk.go | 190 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 410 insertions(+) create mode 100644 v2/config.go create mode 100644 v2/config_test.go create mode 100644 v2/sdk.go diff --git a/v2/config.go b/v2/config.go new file mode 100644 index 0000000000..0b9d232130 --- /dev/null +++ b/v2/config.go @@ -0,0 +1,212 @@ +package nuclei + +import ( + "context" + "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/ratelimit" +) + +// config contains all SDK configuration options +type TemplateFilters struct { + Severity string // filter by severities (accepts CSV values of info, low, medium, high, critical) + ExcludeSeverities string // filter by excluding severities (accepts CSV values of info, low, medium, high, critical) + Authors []string // fiter by author + Tags []string // filter by tags present in template + ExcludeTags []string // filter by excluding tags present in template + IncludeTags []string // filter by including tags present in template + IDs []string // filter by template IDs + ExcludeIDs []string // filter by excluding template IDs + ProtocolTypes []string // filter by protocol types + ExcludeProtocolTypes []string // filter by excluding protocol types + TemplateCondition []string // DSL condition/ expression +} + +// WithTemplateFilters sets template filters and only templates matching the filters will be +// loaded and executed +func WithTemplateFilters(filters TemplateFilters) NucleiSDKOptions { + return func(e *NucleiEngine) error { + s := severity.Severities{} + if err := s.Set(filters.Severity); err != nil { + return err + } + es := severity.Severities{} + if err := es.Set(filters.ExcludeSeverities); err != nil { + return err + } + e.opts.Authors = filters.Authors + e.opts.Tags = filters.Tags + e.opts.ExcludeTags = filters.ExcludeTags + e.opts.IncludeTags = filters.IncludeTags + e.opts.IncludeIds = filters.IDs + e.opts.ExcludeIds = filters.ExcludeIDs + e.opts.Severities = s + e.opts.ExcludeSeverities = es + e.opts.IncludeConditions = filters.TemplateCondition + return nil + } +} + +// InteractshOpts contains options for interactsh +type InteractshOpts interactsh.Options + +// WithInteractshOptions sets interactsh options +func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions { + return func(e *NucleiEngine) error { + optsPtr := &opts + e.interactshOpts = (*interactsh.Options)(optsPtr) + return nil + } +} + +// Concurrency options +type Concurrency struct { + TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode) + HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode) + HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode) + HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode) +} + +// WithConcurrency sets concurrency options +func WithConcurrency(opts Concurrency) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.TemplateThreads = opts.TemplateConcurrency + e.opts.BulkSize = opts.HostConcurrency + e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency + e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency + return nil + } +} + +// WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options +func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration) + return nil + } +} + +type HeadlessOpts struct { + PageTimeout int // timeout for page load + ShowBrowser bool + HeadlessOptions []string + UseChrome bool +} + +// EnableHeadless allows execution of headless templates +// *Use With Caution*: Enabling headless mode may open up attack surface due to browser usage +// and can be prone to exploitation by custom unverified templates if not properly configured +func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.Headless = true + if hopts != nil { + e.opts.HeadlessOptionalArguments = hopts.HeadlessOptions + e.opts.PageTimeout = hopts.PageTimeout + e.opts.ShowBrowser = hopts.ShowBrowser + e.opts.UseInstalledChrome = hopts.UseChrome + } + return nil + } +} + +// EnableStats enables Stats collection with defined interval(in sec) and callback +// Note: callback is executed in a separate goroutine +func EnableStatsWithOpts(interval int, callback func()) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.StatsInterval = interval + return nil + } +} + +// VerbosityOptions +type VerbosityOptions struct { + Verbose bool // show verbose output + Silent bool // show only results + Debug bool // show debug output + DebugRequest bool // show request in debug output + DebugResponse bool // show response in debug output + ShowVarDump bool // show variable dumps in output +} + +// WithVerbosity allows setting verbosity options of (internal) nuclei engine +// and does not affect SDK output +func WithVerbosity(opts VerbosityOptions) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.Verbose = opts.Verbose + e.opts.Silent = opts.Silent + e.opts.Debug = opts.Debug + e.opts.DebugRequests = opts.DebugRequest + e.opts.DebugResponse = opts.DebugResponse + e.opts.ShowVarDump = opts.ShowVarDump + return nil + } +} + +// NetworkConfig contains network config options +// ex: retries , httpx probe , timeout etc +type NetworkConfig struct { + Timeout int // Timeout in seconds + Retries int // Number of retries + LeaveDefaultPorts bool // Leave default ports for http/https + MaxHostError int // Maximum number of host errors to allow before skipping that host + TrackError []string // Adds given errors to max host error watchlist + DisableMaxHostErr bool // Disable max host error optimization (Hosts are not skipped even if they are not responding) +} + +// WithNetworkConfig allows setting network config options +func WithNetworkConfig(opts NetworkConfig) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.Timeout = opts.Timeout + e.opts.Retries = opts.Retries + e.opts.LeaveDefaultPorts = opts.LeaveDefaultPorts + e.hostErrCache = hosterrorscache.New(opts.MaxHostError, hosterrorscache.DefaultMaxHostsCount, opts.TrackError) + return nil + } +} + +// WithProxy allows setting proxy options +func WithProxy(proxy []string, proxyInternalRequests bool) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.Proxy = proxy + e.opts.ProxyInternal = proxyInternalRequests + return nil + } +} + +// WithScanStrategy allows setting scan strategy options +func WithScanStrategy(strategy string) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.ScanStrategy = strategy + return nil + } +} + +// OutputWriter +type OutputWriter output.Writer + +// UseWriter allows setting custom output writer +// by default a mock writer is used with user defined callback +// if outputWriter is used callback will be ignored +func UseOutputWriter(writer OutputWriter) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.customWriter = writer + return nil + } +} + +// StatsWriter +type StatsWriter progress.Progress + +// UseStatsWriter allows setting a custom stats writer +// which can be used to write stats somewhere (ex: send to webserver etc) +func UseStatsWriter(writer StatsWriter) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.customProgress = writer + return nil + } +} diff --git a/v2/config_test.go b/v2/config_test.go new file mode 100644 index 0000000000..02207f5762 --- /dev/null +++ b/v2/config_test.go @@ -0,0 +1,8 @@ +package nuclei_test + +import ( + "testing" +) + +func TestABc(t *testing.T) { +} diff --git a/v2/sdk.go b/v2/sdk.go new file mode 100644 index 0000000000..769afd0fbf --- /dev/null +++ b/v2/sdk.go @@ -0,0 +1,190 @@ +package nuclei + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "time" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/parsers" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/ratelimit" + errorutil "github.com/projectdiscovery/utils/errors" +) + +// NucleiSDKOptions contains options for nuclei SDK +type NucleiSDKOptions func(e *NucleiEngine) error + +// NucleiEngine is the Engine/Client for nuclei which +// runs scans using templates and returns results +type NucleiEngine struct { + // user options + resultCallback func(event *output.ResultEvent) + + // unexported core fields + interactshClient *interactsh.Client + catalog *disk.DiskCatalog + rateLimiter *ratelimit.Limiter + store *loader.Store + httpxClient *httpx.HTTPX + inputProvider *inputs.SimpleInputProvider + engine *core.Engine + + // unexported meta options + opts *types.Options + interactshOpts *interactsh.Options + hostErrCache *hosterrorscache.Cache + customWriter output.Writer + customProgress progress.Progress + rc reporting.Client + executerOpts protocols.ExecutorOptions +} + +// LoadAllTemplates loads all nuclei template based on given options +func (e *NucleiEngine) LoadAllTemplates() error { + workflowLoader, err := parsers.NewLoader(&e.executerOpts) + if err != nil { + return errorutil.New("Could not create workflow loader: %s\n", err) + } + e.executerOpts.WorkflowLoader = workflowLoader + + e.store, err = loader.New(loader.NewConfig(e.opts, e.catalog, e.executerOpts)) + if err != nil { + return errorutil.New("Could not create loader client: %s\n", err) + } + e.store.Load() + return nil +} + +// LoadTargets(urls/domains/ips only) adds targets to the nuclei engine +func (e *NucleiEngine) LoadTargets(targets []string, probeNonHttp bool) { + for _, target := range targets { + if probeNonHttp { + e.inputProvider.SetWithProbe(target, e.httpxClient) + } else { + e.inputProvider.Set(target) + } + } +} + +// LoadTargetsFromReader adds targets(urls/domains/ips only) from reader to the nuclei engine +func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader) { + buff := bufio.NewScanner(reader) + for buff.Scan() { + e.inputProvider.Set(buff.Text()) + } +} + +// applyRequiredDefaults to options +func (e *NucleiEngine) applyRequiredDefaults() { + if e.customWriter == nil { + e.customWriter = testutils.NewMockOutputWriter() + } + if e.customProgress == nil { + e.customProgress = &testutils.MockProgressClient{} + } + if e.hostErrCache == nil { + e.hostErrCache = hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil) + } + // setup interactsh + if e.interactshOpts != nil { + e.interactshOpts.Output = e.customWriter + e.interactshOpts.Progress = e.customProgress + } else { + e.interactshOpts = interactsh.DefaultOptions(e.customWriter, e.rc, e.customProgress) + } + if e.resultCallback == nil { + e.resultCallback = func(event *output.ResultEvent) { + bin, _ := json.Marshal(event) + fmt.Printf("%v\n", string(bin)) + } + } + if e.rateLimiter == nil { + e.rateLimiter = ratelimit.New(context.Background(), 150, time.Second) + } + // these templates are known to have weak matchers + // and idea is to disable them to avoid false positives + e.opts.ExcludeTags = config.ReadIgnoreFile().Tags + + e.inputProvider = &inputs.SimpleInputProvider{ + Inputs: []*contextargs.MetaInput{}, + } +} + +// init +func (e *NucleiEngine) init() error { + protocolstate.Init(e.opts) + protocolinit.Init(e.opts) + e.applyRequiredDefaults() + var err error + + if e.rc, err = reporting.New(&reporting.Options{}, ""); err != nil { + return err + } + e.interactshOpts.IssuesClient = e.rc + if e.interactshClient, err = interactsh.New(e.interactshOpts); err != nil { + return err + } + + e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) + + e.executerOpts = protocols.ExecutorOptions{ + Output: e.customWriter, + Options: e.opts, + Progress: e.customProgress, + Catalog: e.catalog, + IssuesClient: e.rc, + RateLimiter: e.rateLimiter, + Interactsh: e.interactshClient, + HostErrorsCache: e.hostErrCache, + Colorizer: aurora.NewAurora(true), + ResumeCfg: types.NewResumeCfg(), + } + + e.engine = core.New(e.opts) + e.engine.SetExecuterOptions(e.executerOpts) + + httpxOptions := httpx.DefaultOptions + httpxOptions.Timeout = 5 * time.Second + if e.httpxClient, err = httpx.New(&httpxOptions); err != nil { + return err + } + + return nil +} + +// NewNucleiEngine creates a new nuclei engine instance +func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) { + // default options + e := &NucleiEngine{ + opts: types.DefaultOptions(), + } + for _, option := range options { + if err := option(e); err != nil { + return nil, err + } + } + if err := e.init(); err != nil { + return nil, err + } + return e, nil +} From 1b6e6039bb8f4740c83748e105c70af1e40e4ac7 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 03:39:01 +0530 Subject: [PATCH 02/10] nuclei v3 new sdk/library --- .github/workflows/build-test.yml | 10 +- README.md | 2 +- v2/config_test.go | 8 -- v2/examples/advanced/advanced.go | 49 +++++++ v2/examples/simple.go | 106 -------------- v2/examples/simple/simple.go | 20 +++ v2/internal/installer/versioncheck.go | 77 ++++++---- v2/internal/runner/options.go | 4 +- v2/lib/README.md | 87 ++++++++++++ v2/{ => lib}/config.go | 98 ++++++++++++- v2/lib/example_test.go | 76 ++++++++++ v2/lib/helper.go | 33 +++++ v2/lib/multi.go | 152 ++++++++++++++++++++ v2/{ => lib}/sdk.go | 150 +++++++++----------- v2/lib/sdk_private.go | 196 ++++++++++++++++++++++++++ v2/lib/sdk_test.go | 42 ++++++ v2/pkg/testutils/testutils.go | 4 + 17 files changed, 882 insertions(+), 232 deletions(-) delete mode 100644 v2/config_test.go create mode 100644 v2/examples/advanced/advanced.go delete mode 100644 v2/examples/simple.go create mode 100644 v2/examples/simple/simple.go create mode 100644 v2/lib/README.md rename v2/{ => lib}/config.go (69%) create mode 100644 v2/lib/example_test.go create mode 100644 v2/lib/helper.go create mode 100644 v2/lib/multi.go rename v2/{ => lib}/sdk.go (53%) create mode 100644 v2/lib/sdk_private.go create mode 100644 v2/lib/sdk_test.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 8173b810d2..559d7879cc 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -56,6 +56,10 @@ jobs: run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version working-directory: v2/cmd/nuclei/ - - name: Example Code Tests - run: go build . - working-directory: v2/examples/ + - name: Example SDK Simple + run: go run . + working-directory: v2/examples/simple/ + + - name: Example SDK Advanced + run: go run . + working-directory: v2/examples/advanced/ diff --git a/README.md b/README.md index 06845dcdc5..9dee14b183 100644 --- a/README.md +++ b/README.md @@ -379,7 +379,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu ### Using Nuclei From Go Code -Examples of using Nuclei From Go Code to run templates on targets are provided in the [examples](v2/examples/) folder. +Complete guide of using Nuclei as Library/SDK is available at [lib](v2/lib/README.md) ### Resources diff --git a/v2/config_test.go b/v2/config_test.go deleted file mode 100644 index 02207f5762..0000000000 --- a/v2/config_test.go +++ /dev/null @@ -1,8 +0,0 @@ -package nuclei_test - -import ( - "testing" -) - -func TestABc(t *testing.T) { -} diff --git a/v2/examples/advanced/advanced.go b/v2/examples/advanced/advanced.go new file mode 100644 index 0000000000..25dcd00985 --- /dev/null +++ b/v2/examples/advanced/advanced.go @@ -0,0 +1,49 @@ +package main + +import ( + nuclei "github.com/projectdiscovery/nuclei/v2/lib" + "github.com/remeh/sizedwaitgroup" +) + +func main() { + // create nuclei engine with options + ne, err := nuclei.NewThreadSafeNucleiEngine() + if err != nil { + panic(err) + } + // setup sizedWaitgroup to handle concurrency + // here we are using sizedWaitgroup to limit concurrency to 1 + sg := sizedwaitgroup.New(1) + + // scan 1 = run dns templates on scanme.sh + sg.Add() + go func() { + defer sg.Done() + err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), + nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + ) + if err != nil { + panic(err) + } + }() + + // scan 2 = run dns templates on honey.scanme.sh + sg.Add() + go func() { + defer sg.Done() + err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + if err != nil { + panic(err) + } + }() + + // wait for all scans to finish + sg.Wait() + defer ne.Close() + + // Output: + // [dns-saas-service-detection] scanme.sh + // [nameserver-fingerprint] scanme.sh + // [dns-saas-service-detection] honey.scanme.sh +} diff --git a/v2/examples/simple.go b/v2/examples/simple.go deleted file mode 100644 index e1f9a4cd4d..0000000000 --- a/v2/examples/simple.go +++ /dev/null @@ -1,106 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - "os" - "path/filepath" - "time" - - "github.com/logrusorgru/aurora" - - "github.com/projectdiscovery/goflags" - "github.com/projectdiscovery/httpx/common/httpx" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" - "github.com/projectdiscovery/nuclei/v2/pkg/core" - "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/parsers" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/ratelimit" -) - -func main() { - cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil) - defer cache.Close() - - mockProgress := &testutils.MockProgressClient{} - reportingClient, _ := reporting.New(&reporting.Options{}, "") - defer reportingClient.Close() - - outputWriter := testutils.NewMockOutputWriter() - outputWriter.WriteCallback = func(event *output.ResultEvent) { - fmt.Printf("Got Result: %v\n", event) - } - - defaultOpts := types.DefaultOptions() - protocolstate.Init(defaultOpts) - protocolinit.Init(defaultOpts) - - defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"} - defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags - - interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) - interactClient, err := interactsh.New(interactOpts) - if err != nil { - log.Fatalf("Could not create interact client: %s\n", err) - } - defer interactClient.Close() - - home, _ := os.UserHomeDir() - catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates")) - executerOpts := protocols.ExecutorOptions{ - Output: outputWriter, - Options: defaultOpts, - Progress: mockProgress, - Catalog: catalog, - IssuesClient: reportingClient, - RateLimiter: ratelimit.New(context.Background(), 150, time.Second), - Interactsh: interactClient, - HostErrorsCache: cache, - Colorizer: aurora.NewAurora(true), - ResumeCfg: types.NewResumeCfg(), - } - engine := core.New(defaultOpts) - engine.SetExecuterOptions(executerOpts) - - workflowLoader, err := parsers.NewLoader(&executerOpts) - if err != nil { - log.Fatalf("Could not create workflow loader: %s\n", err) - } - executerOpts.WorkflowLoader = workflowLoader - - store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) - if err != nil { - log.Fatalf("Could not create loader client: %s\n", err) - } - store.Load() - - // flat input without probe - inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}} - input := &inputs.SimpleInputProvider{Inputs: inputArgs} - - httpxOptions := httpx.DefaultOptions - httpxOptions.Timeout = 5 * time.Second - httpxClient, err := httpx.New(&httpxOptions) - if err != nil { - log.Fatal(err) - } - - // use httpx to probe the URL => https://scanme.sh - input.SetWithProbe("scanme.sh", httpxClient) - - _ = engine.Execute(store.Templates(), input) - engine.WorkPool().Wait() // Wait for the scan to finish -} diff --git a/v2/examples/simple/simple.go b/v2/examples/simple/simple.go new file mode 100644 index 0000000000..06f9ce2dae --- /dev/null +++ b/v2/examples/simple/simple.go @@ -0,0 +1,20 @@ +package main + +import nuclei "github.com/projectdiscovery/nuclei/v2/lib" + +func main() { + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // only use dns templates + nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + ) + if err != nil { + panic(err) + } + // load targets and optionally probe non http/https targets + ne.LoadTargets([]string{"scanme.sh"}, false) + err = ne.ExecuteWithCallback(nil) + if err != nil { + panic(err) + } + defer ne.Close() +} diff --git a/v2/internal/installer/versioncheck.go b/v2/internal/installer/versioncheck.go index 12dd2aed51..88857410da 100644 --- a/v2/internal/installer/versioncheck.go +++ b/v2/internal/installer/versioncheck.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "runtime" + "sync" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/retryablehttp-go" @@ -34,7 +35,55 @@ type PdtmAPIResponse struct { // and returns an error if it fails to check on success it returns nil and changes are // made to the default config in config.DefaultConfig func NucleiVersionCheck() error { - resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams()) + return doVersionCheck(false) +} + +// this will be updated by features of 1.21 release (which directly provides sync.Once(func())) +type sdkUpdateCheck struct { + sync.Once +} + +var sdkUpdateCheckInstance = &sdkUpdateCheck{} + +// NucleiSDKVersionCheck checks for latest version of nuclei which running in sdk mode +// this only happens once per process regardless of how many times this function is called +func NucleiSDKVersionCheck() { + sdkUpdateCheckInstance.Do(func() { + _ = doVersionCheck(true) + }) +} + +// getpdtmParams returns encoded query parameters sent to update check endpoint +func getpdtmParams(isSDK bool) string { + params := &url.Values{} + params.Add("os", runtime.GOOS) + params.Add("arch", runtime.GOARCH) + params.Add("go_version", runtime.Version()) + params.Add("v", config.Version) + if isSDK { + params.Add("sdk", "true") + } + return params.Encode() +} + +// UpdateIgnoreFile updates default ignore file by downloading latest ignore file +func UpdateIgnoreFile() error { + resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams(false)) + if err != nil { + return err + } + bin, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil { + return err + } + return config.DefaultConfig.UpdateNucleiIgnoreHash() +} + +func doVersionCheck(isSDK bool) error { + resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK)) if err != nil { return err } @@ -63,29 +112,3 @@ func NucleiVersionCheck() error { } return config.DefaultConfig.WriteVersionCheckData(pdtmResp.IgnoreHash, nucleiversion, templateversion) } - -// getpdtmParams returns encoded query parameters sent to update check endpoint -func getpdtmParams() string { - params := &url.Values{} - params.Add("os", runtime.GOOS) - params.Add("arch", runtime.GOARCH) - params.Add("go_version", runtime.Version()) - params.Add("v", config.Version) - return params.Encode() -} - -// UpdateIgnoreFile updates default ignore file by downloading latest ignore file -func UpdateIgnoreFile() error { - resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams()) - if err != nil { - return err - } - bin, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil { - return err - } - return config.DefaultConfig.UpdateNucleiIgnoreHash() -} diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 831842c72e..51ecace493 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -71,7 +71,7 @@ func ParseOptions(options *types.Options) { } // Validate the options passed by the user and if any // invalid options have been used, exit. - if err := validateOptions(options); err != nil { + if err := ValidateOptions(options); err != nil { gologger.Fatal().Msgf("Program exiting: %s\n", err) } @@ -105,7 +105,7 @@ func ParseOptions(options *types.Options) { } // validateOptions validates the configuration options passed -func validateOptions(options *types.Options) error { +func ValidateOptions(options *types.Options) error { validate := validator.New() if err := validate.Struct(options); err != nil { if _, ok := err.(*validator.InvalidValidationError); ok { diff --git a/v2/lib/README.md b/v2/lib/README.md new file mode 100644 index 0000000000..3649e54204 --- /dev/null +++ b/v2/lib/README.md @@ -0,0 +1,87 @@ +## Using Nuclei as Library + +Nuclei was primarily built as a CLI tool, but with increasing choice of users wanting to use nuclei as library in their own automation, we have added a simplified Library/SDK of nuclei in v3 + +### Installation + +To add nuclei as a library to your go project, you can use the following command: + +```bash +go get -u github.com/projectdiscovery/nuclei/v2/lib +``` + +Or add below import to your go file and let IDE handle the rest: + +```go +import nuclei "github.com/projectdiscovery/nuclei/v2/lib" +``` + +## Basic Example of using Nuclei Library/SDK + +```go +// create nuclei engine with options + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{Severity: "critical"}), // run critical severity templates only + ) + if err != nil { + panic(err) + } + // load targets and optionally probe non http/https targets + ne.LoadTargets([]string{"scanme.sh"}, false) + err = ne.ExecuteWithCallback(nil) + if err != nil { + panic(err) + } + defer ne.Close() +``` + +## Advanced Example of using Nuclei Library/SDK + +For Various use cases like batching etc you might want to run nuclei in goroutines this can be done by using `nuclei.NewThreadSafeNucleiEngine` + +```go +// create nuclei engine with options + ne, err := nuclei.NewThreadSafeNucleiEngine() + if err != nil{ + panic(err) + } + // setup waitgroup to handle concurrency + wg := &sync.WaitGroup{} + + // scan 1 = run dns templates on scanme.sh + wg.Add(1) + go func() { + defer wg.Done() + err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + if err != nil { + panic(err) + } + }() + + // scan 2 = run http templates on honey.scanme.sh + wg.Add(1) + go func() { + defer wg.Done() + err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + if err != nil { + panic(err) + } + }() + + // wait for all scans to finish + wg.Wait() + defer ne.Close() +``` + +## More Documentation + +For complete documentation of nuclei library, please refer to [godoc](https://pkg.go.dev/github.com/projectdiscovery/nuclei/v2/lib) which contains all available options and methods. + + + +### Note + +| :exclamation: **Disclaimer** | +|---------------------------------| +| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. | +| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. | \ No newline at end of file diff --git a/v2/config.go b/v2/lib/config.go similarity index 69% rename from v2/config.go rename to v2/lib/config.go index 0b9d232130..89b34cf6d3 100644 --- a/v2/config.go +++ b/v2/lib/config.go @@ -4,11 +4,15 @@ import ( "context" "time" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/ratelimit" ) @@ -16,14 +20,14 @@ import ( type TemplateFilters struct { Severity string // filter by severities (accepts CSV values of info, low, medium, high, critical) ExcludeSeverities string // filter by excluding severities (accepts CSV values of info, low, medium, high, critical) + ProtocolTypes string // filter by protocol types + ExcludeProtocolTypes string // filter by excluding protocol types Authors []string // fiter by author Tags []string // filter by tags present in template ExcludeTags []string // filter by excluding tags present in template IncludeTags []string // filter by including tags present in template IDs []string // filter by template IDs ExcludeIDs []string // filter by excluding template IDs - ProtocolTypes []string // filter by protocol types - ExcludeProtocolTypes []string // filter by excluding protocol types TemplateCondition []string // DSL condition/ expression } @@ -39,6 +43,14 @@ func WithTemplateFilters(filters TemplateFilters) NucleiSDKOptions { if err := es.Set(filters.ExcludeSeverities); err != nil { return err } + pt := types.ProtocolTypes{} + if err := pt.Set(filters.ProtocolTypes); err != nil { + return err + } + ept := types.ProtocolTypes{} + if err := ept.Set(filters.ExcludeProtocolTypes); err != nil { + return err + } e.opts.Authors = filters.Authors e.opts.Tags = filters.Tags e.opts.ExcludeTags = filters.ExcludeTags @@ -47,6 +59,8 @@ func WithTemplateFilters(filters TemplateFilters) NucleiSDKOptions { e.opts.ExcludeIds = filters.ExcludeIDs e.opts.Severities = s e.opts.ExcludeSeverities = es + e.opts.Protocols = pt + e.opts.ExcludeProtocols = ept e.opts.IncludeConditions = filters.TemplateCondition return nil } @@ -58,6 +72,9 @@ type InteractshOpts interactsh.Options // WithInteractshOptions sets interactsh options func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions { return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("WithInteractshOptions") + } optsPtr := &opts e.interactshOpts = (*interactsh.Options)(optsPtr) return nil @@ -91,6 +108,7 @@ func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions } } +// HeadlessOpts contains options for headless templates type HeadlessOpts struct { PageTimeout int // timeout for page load ShowBrowser bool @@ -110,15 +128,41 @@ func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions { e.opts.ShowBrowser = hopts.ShowBrowser e.opts.UseInstalledChrome = hopts.UseChrome } + if engine.MustDisableSandbox() { + gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n") + } + browser, err := engine.New(e.opts) + if err != nil { + return err + } + e.executerOpts.Browser = browser return nil } } +// StatsOptions +type StatsOptions struct { + Interval int + JSON bool + StartMetricServer bool + MetricServerPort int +} + // EnableStats enables Stats collection with defined interval(in sec) and callback // Note: callback is executed in a separate goroutine -func EnableStatsWithOpts(interval int, callback func()) NucleiSDKOptions { +func EnableStatsWithOpts(opts StatsOptions) NucleiSDKOptions { return func(e *NucleiEngine) error { - e.opts.StatsInterval = interval + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("EnableStatsWithOpts") + } + if opts.Interval == 0 { + opts.Interval = 5 //sec + } + e.opts.StatsInterval = opts.Interval + e.enableStats = true + e.opts.StatsJSON = opts.JSON + e.opts.Metrics = opts.StartMetricServer + e.opts.MetricsPort = opts.MetricServerPort return nil } } @@ -137,12 +181,17 @@ type VerbosityOptions struct { // and does not affect SDK output func WithVerbosity(opts VerbosityOptions) NucleiSDKOptions { return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("WithVerbosity") + } e.opts.Verbose = opts.Verbose e.opts.Silent = opts.Silent e.opts.Debug = opts.Debug e.opts.DebugRequests = opts.DebugRequest e.opts.DebugResponse = opts.DebugResponse - e.opts.ShowVarDump = opts.ShowVarDump + if opts.ShowVarDump { + vardump.EnableVarDump = true + } return nil } } @@ -161,6 +210,9 @@ type NetworkConfig struct { // WithNetworkConfig allows setting network config options func WithNetworkConfig(opts NetworkConfig) NucleiSDKOptions { return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("WithNetworkConfig") + } e.opts.Timeout = opts.Timeout e.opts.Retries = opts.Retries e.opts.LeaveDefaultPorts = opts.LeaveDefaultPorts @@ -172,6 +224,9 @@ func WithNetworkConfig(opts NetworkConfig) NucleiSDKOptions { // WithProxy allows setting proxy options func WithProxy(proxy []string, proxyInternalRequests bool) NucleiSDKOptions { return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("WithProxy") + } e.opts.Proxy = proxy e.opts.ProxyInternal = proxyInternalRequests return nil @@ -194,6 +249,9 @@ type OutputWriter output.Writer // if outputWriter is used callback will be ignored func UseOutputWriter(writer OutputWriter) NucleiSDKOptions { return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("UseOutputWriter") + } e.customWriter = writer return nil } @@ -206,7 +264,37 @@ type StatsWriter progress.Progress // which can be used to write stats somewhere (ex: send to webserver etc) func UseStatsWriter(writer StatsWriter) NucleiSDKOptions { return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("UseStatsWriter") + } e.customProgress = writer return nil } } + +// WithTemplateUpdateCallback allows setting a callback which will be called +// when nuclei templates are outdated +// Note: Nuclei-templates are crucial part of nuclei and using outdated templates or nuclei sdk is not recommended +// as it may cause unexpected results due to compatibility issues +func WithTemplateUpdateCallback(disableTemplatesAutoUpgrade bool, callback func(newVersion string)) NucleiSDKOptions { + return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("WithTemplateUpdateCallback") + } + e.disableTemplatesAutoUpgrade = disableTemplatesAutoUpgrade + e.onUpdateAvailableCallback = callback + return nil + } +} + +// WithSandboxOptions allows setting supported sandbox options +func WithSandboxOptions(allowLocalFileAccess bool, restrictLocalNetworkAccess bool) NucleiSDKOptions { + return func(e *NucleiEngine) error { + if e.mode == threadSafe { + return ErrOptionsNotSupported.Msgf("WithSandboxOptions") + } + e.opts.AllowLocalFileAccess = allowLocalFileAccess + e.opts.RestrictLocalNetworkAccess = restrictLocalNetworkAccess + return nil + } +} diff --git a/v2/lib/example_test.go b/v2/lib/example_test.go new file mode 100644 index 0000000000..0e15229eb3 --- /dev/null +++ b/v2/lib/example_test.go @@ -0,0 +1,76 @@ +//go:build !race +// +build !race + +package nuclei_test + +import ( + nuclei "github.com/projectdiscovery/nuclei/v2/lib" + "github.com/remeh/sizedwaitgroup" +) + +// A very simple example on how to use nuclei engine +func ExampleNucleiEngine() { + // create nuclei engine with options + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // only use dns templates + nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + ) + if err != nil { + panic(err) + } + // load targets and optionally probe non http/https targets + ne.LoadTargets([]string{"scanme.sh"}, false) + // when callback is nil it nuclei will print JSON output to stdout + err = ne.ExecuteWithCallback(nil) + if err != nil { + panic(err) + } + defer ne.Close() + + // Output: + // [dns-saas-service-detection] scanme.sh + // [nameserver-fingerprint] scanme.sh +} + +func ExampleThreadSafeNucleiEngine() { + // create nuclei engine with options + ne, err := nuclei.NewThreadSafeNucleiEngine() + if err != nil { + panic(err) + } + // setup sizedWaitgroup to handle concurrency + // here we are using sizedWaitgroup to limit concurrency to 1 + sg := sizedwaitgroup.New(1) + + // scan 1 = run dns templates on scanme.sh + sg.Add() + go func() { + defer sg.Done() + err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), + nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + ) + if err != nil { + panic(err) + } + }() + + // scan 2 = run dns templates on honey.scanme.sh + sg.Add() + go func() { + defer sg.Done() + err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + if err != nil { + panic(err) + } + }() + + // wait for all scans to finish + sg.Wait() + defer ne.Close() + + // Output: + // [dns-saas-service-detection] scanme.sh + // [nameserver-fingerprint] scanme.sh + // [dns-saas-service-detection] honey.scanme.sh +} diff --git a/v2/lib/helper.go b/v2/lib/helper.go new file mode 100644 index 0000000000..281f6799dc --- /dev/null +++ b/v2/lib/helper.go @@ -0,0 +1,33 @@ +package nuclei + +import ( + "context" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + uncoverNuclei "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/uncover" +) + +// helper.go file proxy execution of all nuclei functions that are nested deep inside multiple packages +// but are helpful / useful while using nuclei as a library + +// GetTargetsFromUncover returns targets from uncover in given format . +// supported formats are any string with [ip,host,port,url] placeholders +func GetTargetsFromUncover(ctx context.Context, outputFormat string, opts *uncover.Options) (chan string, error) { + return uncoverNuclei.GetTargetsFromUncover(ctx, outputFormat, opts) +} + +// GetTargetsFromTemplateMetadata returns all targets by querying engine metadata (ex: fofo-query,shodan-query) etc from given templates . +// supported formats are any string with [ip,host,port,url] placeholders +func GetTargetsFromTemplateMetadata(ctx context.Context, templates []*templates.Template, outputFormat string, opts *uncover.Options) chan string { + return uncoverNuclei.GetUncoverTargetsFromMetadata(ctx, templates, outputFormat, opts) +} + +// DefaultConfig is instance of default nuclei configs +// any mutations to this config will be reflected in all nuclei instances (saves some config to disk) +var DefaultConfig *config.Config + +func init() { + DefaultConfig = config.DefaultConfig +} diff --git a/v2/lib/multi.go b/v2/lib/multi.go new file mode 100644 index 0000000000..ebd85e0542 --- /dev/null +++ b/v2/lib/multi.go @@ -0,0 +1,152 @@ +package nuclei + +import ( + "context" + "time" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/parsers" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/ratelimit" + errorutil "github.com/projectdiscovery/utils/errors" +) + +// unsafeOptions are those nuclei objects/instances/types +// that are required to run nuclei engine but are not thread safe +// hence they are ephemeral and are created on every ExecuteNucleiWithOpts invocation +// in ThreadSafeNucleiEngine +type unsafeOptions struct { + executerOpts protocols.ExecutorOptions + engine *core.Engine +} + +// createEphemeralObjects creates ephemeral nuclei objects/instances/types +func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOptions, error) { + u := &unsafeOptions{} + u.executerOpts = protocols.ExecutorOptions{ + Output: base.customWriter, + Options: opts, + Progress: base.customProgress, + Catalog: base.catalog, + IssuesClient: base.rc, + RateLimiter: base.rateLimiter, + Interactsh: base.interactshClient, + HostErrorsCache: base.hostErrCache, + Colorizer: aurora.NewAurora(true), + ResumeCfg: types.NewResumeCfg(), + } + if opts.RateLimitMinute > 0 { + u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute) + } else if opts.RateLimit > 0 { + u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second) + } else { + u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background()) + } + u.engine = core.New(opts) + u.engine.SetExecuterOptions(u.executerOpts) + return u, nil +} + +// ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe +// and can be used concurrently. Non-thread-safe methods start with Global prefix +type ThreadSafeNucleiEngine struct { + eng *NucleiEngine +} + +// NewThreadSafeNucleiEngine creates a new nuclei engine with given options +// whose methods are thread-safe and can be used concurrently +// Note: Non-thread-safe methods start with Global prefix +func NewThreadSafeNucleiEngine(opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) { + // default options + e := &NucleiEngine{ + opts: types.DefaultOptions(), + mode: threadSafe, + } + for _, option := range opts { + if err := option(e); err != nil { + return nil, err + } + } + if err := e.init(); err != nil { + return nil, err + } + return &ThreadSafeNucleiEngine{eng: e}, nil +} + +// GlobalLoadAllTemplates loads all templates from nuclei-templates repo +// This method will load all templates based on filters given at the time of nuclei engine creation in opts +func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error { + return e.eng.LoadAllTemplates() +} + +// GlobalResultCallback sets a callback function which will be called for each result +func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) { + e.eng.resultCallbacks = []func(*output.ResultEvent){callback} +} + +// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) +// This method can be called concurrently and it will use some global resources but can be runned parllely +// by invoking this method with different options and targets +// Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options +func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error { + baseOpts := *e.eng.opts + tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe} + for _, option := range opts { + if err := option(tmpEngine); err != nil { + return err + } + } + // create ephemeral nuclei objects/instances/types using base nuclei engine + unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts) + if err != nil { + return err + } + + // load templates + workflowLoader, err := parsers.NewLoader(&unsafeOpts.executerOpts) + if err != nil { + return errorutil.New("Could not create workflow loader: %s\n", err) + } + unsafeOpts.executerOpts.WorkflowLoader = workflowLoader + + store, err := loader.New(loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts)) + if err != nil { + return errorutil.New("Could not create loader client: %s\n", err) + } + store.Load() + + inputProvider := &inputs.SimpleInputProvider{ + Inputs: []*contextargs.MetaInput{}, + } + + // load targets + for _, target := range targets { + inputProvider.Set(target) + } + + if len(store.Templates()) == 0 && len(store.Workflows()) == 0 { + return ErrNoTemplatesAvailable + } + if inputProvider.Count() == 0 { + return ErrNoTargetsAvailable + } + + engine := core.New(tmpEngine.opts) + engine.SetExecuterOptions(unsafeOpts.executerOpts) + + _ = engine.ExecuteScanWithOpts(store.Templates(), inputProvider, false) + + engine.WorkPool().Wait() + return nil +} + +// Close all resources used by nuclei engine +func (e *ThreadSafeNucleiEngine) Close() { + e.eng.Close() +} diff --git a/v2/sdk.go b/v2/lib/sdk.go similarity index 53% rename from v2/sdk.go rename to v2/lib/sdk.go index 769afd0fbf..6c1e319b51 100644 --- a/v2/sdk.go +++ b/v2/lib/sdk.go @@ -2,15 +2,9 @@ package nuclei import ( "bufio" - "context" - "encoding/json" - "fmt" "io" - "time" - "github.com/logrusorgru/aurora" "github.com/projectdiscovery/httpx/common/httpx" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" @@ -19,26 +13,50 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" ) // NucleiSDKOptions contains options for nuclei SDK type NucleiSDKOptions func(e *NucleiEngine) error +var ( + // ErrNotImplemented is returned when a feature is not implemented + ErrNotImplemented = errorutil.New("Not implemented") + // ErrNoTemplatesAvailable is returned when no templates are available to execute + ErrNoTemplatesAvailable = errorutil.New("No templates available") + // ErrNoTargetsAvailable is returned when no targets are available to scan + ErrNoTargetsAvailable = errorutil.New("No targets available") + // ErrOptionsNotSupported is returned when an option is not supported in thread safe mode + ErrOptionsNotSupported = errorutil.NewWithFmt("Option %v not supported in thread safe mode") +) + +type engineMode uint + +const ( + singleInstance engineMode = iota + threadSafe +) + // NucleiEngine is the Engine/Client for nuclei which // runs scans using templates and returns results type NucleiEngine struct { // user options - resultCallback func(event *output.ResultEvent) + resultCallbacks []func(event *output.ResultEvent) + onFailureCallback func(event *output.InternalEvent) + disableTemplatesAutoUpgrade bool + enableStats bool + onUpdateAvailableCallback func(newVersion string) + + // ready-status fields + templatesLoaded bool // unexported core fields interactshClient *interactsh.Client @@ -48,6 +66,9 @@ type NucleiEngine struct { httpxClient *httpx.HTTPX inputProvider *inputs.SimpleInputProvider engine *core.Engine + mode engineMode + browserInstance *engine.Browser + httpClient *retryablehttp.Client // unexported meta options opts *types.Options @@ -75,6 +96,14 @@ func (e *NucleiEngine) LoadAllTemplates() error { return nil } +// GetTemplates returns all nuclei templates that are loaded +func (e *NucleiEngine) GetTemplates() []*templates.Template { + if !e.templatesLoaded { + _ = e.LoadAllTemplates() + } + return e.store.Templates() +} + // LoadTargets(urls/domains/ips only) adds targets to the nuclei engine func (e *NucleiEngine) LoadTargets(targets []string, probeNonHttp bool) { for _, target := range targets { @@ -87,88 +116,48 @@ func (e *NucleiEngine) LoadTargets(targets []string, probeNonHttp bool) { } // LoadTargetsFromReader adds targets(urls/domains/ips only) from reader to the nuclei engine -func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader) { +func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader, probeNonHttp bool) { buff := bufio.NewScanner(reader) for buff.Scan() { - e.inputProvider.Set(buff.Text()) - } -} - -// applyRequiredDefaults to options -func (e *NucleiEngine) applyRequiredDefaults() { - if e.customWriter == nil { - e.customWriter = testutils.NewMockOutputWriter() - } - if e.customProgress == nil { - e.customProgress = &testutils.MockProgressClient{} - } - if e.hostErrCache == nil { - e.hostErrCache = hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil) - } - // setup interactsh - if e.interactshOpts != nil { - e.interactshOpts.Output = e.customWriter - e.interactshOpts.Progress = e.customProgress - } else { - e.interactshOpts = interactsh.DefaultOptions(e.customWriter, e.rc, e.customProgress) - } - if e.resultCallback == nil { - e.resultCallback = func(event *output.ResultEvent) { - bin, _ := json.Marshal(event) - fmt.Printf("%v\n", string(bin)) + if probeNonHttp { + e.inputProvider.SetWithProbe(buff.Text(), e.httpxClient) + } else { + e.inputProvider.Set(buff.Text()) } } - if e.rateLimiter == nil { - e.rateLimiter = ratelimit.New(context.Background(), 150, time.Second) - } - // these templates are known to have weak matchers - // and idea is to disable them to avoid false positives - e.opts.ExcludeTags = config.ReadIgnoreFile().Tags - - e.inputProvider = &inputs.SimpleInputProvider{ - Inputs: []*contextargs.MetaInput{}, - } } -// init -func (e *NucleiEngine) init() error { - protocolstate.Init(e.opts) - protocolinit.Init(e.opts) - e.applyRequiredDefaults() - var err error +// Close all resources used by nuclei engine +func (e *NucleiEngine) Close() { + e.interactshClient.Close() + e.rc.Close() + e.customWriter.Close() + e.hostErrCache.Close() + e.executerOpts.RateLimiter.Stop() +} - if e.rc, err = reporting.New(&reporting.Options{}, ""); err != nil { - return err +// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) +func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error { + if !e.templatesLoaded { + _ = e.LoadAllTemplates() } - e.interactshOpts.IssuesClient = e.rc - if e.interactshClient, err = interactsh.New(e.interactshOpts); err != nil { - return err + if len(e.store.Templates()) == 0 && len(e.store.Workflows()) == 0 { + return ErrNoTemplatesAvailable } - - e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) - - e.executerOpts = protocols.ExecutorOptions{ - Output: e.customWriter, - Options: e.opts, - Progress: e.customProgress, - Catalog: e.catalog, - IssuesClient: e.rc, - RateLimiter: e.rateLimiter, - Interactsh: e.interactshClient, - HostErrorsCache: e.hostErrCache, - Colorizer: aurora.NewAurora(true), - ResumeCfg: types.NewResumeCfg(), + if e.inputProvider.Count() == 0 { + return ErrNoTargetsAvailable } - e.engine = core.New(e.opts) - e.engine.SetExecuterOptions(e.executerOpts) - - httpxOptions := httpx.DefaultOptions - httpxOptions.Timeout = 5 * time.Second - if e.httpxClient, err = httpx.New(&httpxOptions); err != nil { - return err + filtered := []func(event *output.ResultEvent){} + for _, callback := range callback { + if callback != nil { + filtered = append(filtered, callback) + } } + e.resultCallbacks = append(e.resultCallbacks, filtered...) + _ = e.engine.ExecuteScanWithOpts(e.store.Templates(), e.inputProvider, false) + defer e.engine.WorkPool().Wait() return nil } @@ -177,6 +166,7 @@ func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) { // default options e := &NucleiEngine{ opts: types.DefaultOptions(), + mode: singleInstance, } for _, option := range options { if err := option(e); err != nil { diff --git a/v2/lib/sdk_private.go b/v2/lib/sdk_private.go new file mode 100644 index 0000000000..5703ed2451 --- /dev/null +++ b/v2/lib/sdk_private.go @@ -0,0 +1,196 @@ +package nuclei + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v2/internal/installer" + "github.com/projectdiscovery/nuclei/v2/internal/runner" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/core" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/ratelimit" +) + +// applyRequiredDefaults to options +func (e *NucleiEngine) applyRequiredDefaults() { + if e.customWriter == nil { + mockoutput := testutils.NewMockOutputWriter() + mockoutput.WriteCallback = func(event *output.ResultEvent) { + if len(e.resultCallbacks) > 0 { + for _, callback := range e.resultCallbacks { + if callback != nil { + callback(event) + } + } + return + } + sb := strings.Builder{} + sb.WriteString(fmt.Sprintf("[%v] ", event.TemplateID)) + if event.Matched != "" { + sb.WriteString(event.Matched) + } else { + sb.WriteString(event.Host) + } + fmt.Println(sb.String()) + } + if e.onFailureCallback != nil { + mockoutput.FailureCallback = e.onFailureCallback + } + e.customWriter = mockoutput + } + if e.customProgress == nil { + e.customProgress = &testutils.MockProgressClient{} + } + if e.hostErrCache == nil { + e.hostErrCache = hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil) + } + // setup interactsh + if e.interactshOpts != nil { + e.interactshOpts.Output = e.customWriter + e.interactshOpts.Progress = e.customProgress + } else { + e.interactshOpts = interactsh.DefaultOptions(e.customWriter, e.rc, e.customProgress) + } + if e.rateLimiter == nil { + e.rateLimiter = ratelimit.New(context.Background(), 150, time.Second) + } + // these templates are known to have weak matchers + // and idea is to disable them to avoid false positives + e.opts.ExcludeTags = config.ReadIgnoreFile().Tags + + e.inputProvider = &inputs.SimpleInputProvider{ + Inputs: []*contextargs.MetaInput{}, + } +} + +// init +func (e *NucleiEngine) init() error { + if e.opts.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + } else if e.opts.Debug { + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + } else if e.opts.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } + + if err := runner.ValidateOptions(e.opts); err != nil { + return err + } + + if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { + httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) + if err != nil { + return err + } + e.httpClient = httpclient + } + + _ = protocolstate.Init(e.opts) + _ = protocolinit.Init(e.opts) + e.applyRequiredDefaults() + var err error + + // setup progressbar + if e.enableStats { + progressInstance, progressErr := progress.NewStatsTicker(e.opts.StatsInterval, e.enableStats, e.opts.StatsJSON, e.opts.Metrics, false, e.opts.MetricsPort) + if progressErr != nil { + return err + } + e.customProgress = progressInstance + } + + if err := reporting.CreateConfigIfNotExists(); err != nil { + return err + } + // we don't support reporting config in sdk mode + if e.rc, err = reporting.New(&reporting.Options{}, ""); err != nil { + return err + } + e.interactshOpts.IssuesClient = e.rc + if e.httpClient != nil { + e.interactshOpts.HTTPClient = e.httpClient + } + if e.interactshClient, err = interactsh.New(e.interactshOpts); err != nil { + return err + } + + e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) + + e.executerOpts = protocols.ExecutorOptions{ + Output: e.customWriter, + Options: e.opts, + Progress: e.customProgress, + Catalog: e.catalog, + IssuesClient: e.rc, + RateLimiter: e.rateLimiter, + Interactsh: e.interactshClient, + HostErrorsCache: e.hostErrCache, + Colorizer: aurora.NewAurora(true), + ResumeCfg: types.NewResumeCfg(), + Browser: e.browserInstance, + } + + if e.opts.RateLimitMinute > 0 { + e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute) + } else if e.opts.RateLimit > 0 { + e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second) + } else { + e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background()) + } + + e.engine = core.New(e.opts) + e.engine.SetExecuterOptions(e.executerOpts) + + httpxOptions := httpx.DefaultOptions + httpxOptions.Timeout = 5 * time.Second + if e.httpxClient, err = httpx.New(&httpxOptions); err != nil { + return err + } + + // Only Happens once regardless how many times this function is called + // This will update ignore file to filter out templates with weak matchers to avoid false positives + // and also upgrade templates to latest version if available + installer.NucleiSDKVersionCheck() + + return e.processUpdateCheckResults() +} + +type syncOnce struct { + sync.Once +} + +var updateCheckInstance = &syncOnce{} + +// processUpdateCheckResults processes update check results +func (e *NucleiEngine) processUpdateCheckResults() error { + var err error + updateCheckInstance.Do(func() { + if e.onUpdateAvailableCallback != nil { + e.onUpdateAvailableCallback(config.DefaultConfig.LatestNucleiTemplatesVersion) + } + tm := installer.TemplateManager{} + err = tm.UpdateIfOutdated() + }) + return err +} diff --git a/v2/lib/sdk_test.go b/v2/lib/sdk_test.go new file mode 100644 index 0000000000..8e36af3be2 --- /dev/null +++ b/v2/lib/sdk_test.go @@ -0,0 +1,42 @@ +package nuclei_test + +import ( + "testing" + + nuclei "github.com/projectdiscovery/nuclei/v2/lib" + "github.com/stretchr/testify/require" +) + +func TestExample(t *testing.T) { + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), + nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), + ) + require.Nil(t, err) + ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here + // when callback is nil it nuclei will print JSON output to stdout + err = ne.ExecuteWithCallback(nil) + require.Nil(t, err) + defer ne.Close() +} + +func TestThreadSafeNuclei(t *testing.T) { + // create nuclei engine with options + ne, err := nuclei.NewThreadSafeNucleiEngine() + require.Nil(t, err) + + // scan 1 = run dns templates on scanme.sh + t.Run("scanme.sh", func(t *testing.T) { + err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + require.Nil(t, err) + }) + + // scan 2 = run dns templates on honey.scanme.sh + t.Run("honey.scanme.sh", func(t *testing.T) { + err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + require.Nil(t, err) + }) + + // wait for all scans to finish + defer ne.Close() +} diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 616e0c5ebf..7f0b16a377 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -106,6 +106,7 @@ func (n *NoopWriter) Write(data []byte, level levels.Level) {} type MockOutputWriter struct { aurora aurora.Aurora RequestCallback func(templateID, url, requestType string, err error) + FailureCallback func(result *output.InternalEvent) WriteCallback func(o *output.ResultEvent) } @@ -139,6 +140,9 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro // WriteFailure writes the event to file and/or screen. func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error { + if m.FailureCallback != nil && result != nil { + m.FailureCallback(&result) + } return nil } func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) { From 6b62488bdb467009337bace979c7b409a623d5ec Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 03:52:55 +0530 Subject: [PATCH 03/10] fix TestActionGetResource broken link --- v2/pkg/protocols/headless/engine/page_actions_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3524b47d86..7930598d21 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -349,7 +349,7 @@ func TestActionGetResource(t *testing.T) { Nuclei Test Page - + ` @@ -360,7 +360,7 @@ func TestActionGetResource(t *testing.T) { testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") - require.Equal(t, len(out["src"]), 3159, "could not find resource") + require.Equal(t, len(out["src"]), 121808, "could not find resource") }) } From 2b5378f08e03ba6064ac77b970b35a10402f344a Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 19:03:36 +0530 Subject: [PATCH 04/10] fix clistats + clustering and more --- v2/cmd/nuclei/main.go | 1 - v2/examples/simple/simple.go | 6 ++-- v2/internal/runner/runner.go | 41 +++----------------------- v2/lib/README.md | 2 +- v2/lib/config.go | 8 ++--- v2/lib/example_test.go | 23 ++++++++++----- v2/lib/sdk_private.go | 3 +- v2/lib/sdk_test.go | 2 +- v2/pkg/core/execute_options.go | 40 ++++++++++++++++++++++++- v2/pkg/core/workflow_execute_test.go | 12 ++++---- v2/pkg/progress/progress.go | 38 +++++------------------- v2/pkg/protocols/multi/request_test.go | 2 +- v2/pkg/templates/compile_test.go | 2 +- v2/pkg/testutils/testutils.go | 2 +- v2/pkg/types/types.go | 2 +- 15 files changed, 86 insertions(+), 98 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 1554eea1dd..e574f31369 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -361,7 +361,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), - flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"), flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"), ) diff --git a/v2/examples/simple/simple.go b/v2/examples/simple/simple.go index 06f9ce2dae..e9a881ddac 100644 --- a/v2/examples/simple/simple.go +++ b/v2/examples/simple/simple.go @@ -4,14 +4,14 @@ import nuclei "github.com/projectdiscovery/nuclei/v2/lib" func main() { ne, err := nuclei.NewNucleiEngine( - nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // only use dns templates - nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}), + nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), // optionally enable metrics server for better observability ) if err != nil { panic(err) } // load targets and optionally probe non http/https targets - ne.LoadTargets([]string{"scanme.sh"}, false) + ne.LoadTargets([]string{"http://honey.scanme.sh"}, false) err = ne.ExecuteWithCallback(nil) if err != nil { panic(err) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 01c49c9e95..a742bd9cf4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -260,7 +260,7 @@ func New(options *types.Options) (*Runner, error) { statsInterval = -1 options.EnableProgressBar = true } - runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.Cloud, options.MetricsPort) + runner.progress, progressErr = progress.NewStatsTicker(statsInterval, options.EnableProgressBar, options.StatsJSON, options.Cloud, options.MetricsPort) if progressErr != nil { return nil, progressErr } @@ -662,16 +662,6 @@ func (r *Runner) executeSmartWorkflowInput(executorOpts protocols.ExecutorOption } func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) (*atomic.Bool, error) { - var unclusteredRequests int64 - for _, template := range store.Templates() { - // workflows will dynamically adjust the totals while running, as - // it can't be known in advance which requests will be called - if len(template.Workflows) > 0 { - continue - } - unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count() - } - if r.options.VerboseVerbose { for _, template := range store.Templates() { r.logAvailableTemplate(template.Path) @@ -681,34 +671,11 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) } } - // Cluster the templates first because we want info on how many - // templates did we cluster for showing to user in CLI - originalTemplatesCount := len(store.Templates()) - finalTemplates, clusterCount := templates.ClusterTemplates(store.Templates(), engine.ExecuterOptions()) + finalTemplates := []*templates.Template{} + finalTemplates = append(finalTemplates, store.Templates()...) finalTemplates = append(finalTemplates, store.Workflows()...) - var totalRequests int64 - for _, t := range finalTemplates { - if len(t.Workflows) > 0 { - continue - } - totalRequests += int64(t.Executer.Requests()) * r.hmapInputProvider.Count() - } - if totalRequests < unclusteredRequests { - gologger.Info().Msgf("Templates clustered: %d (Reduced %d Requests)", clusterCount, unclusteredRequests-totalRequests) - } - workflowCount := len(store.Workflows()) - templateCount := originalTemplatesCount + workflowCount - - // 0 matches means no templates were found in the directory - if templateCount == 0 { - return &atomic.Bool{}, errors.New("no valid templates were found") - } - - // tracks global progress and captures stdout/stderr until p.Wait finishes - r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests) - - results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, true) + results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, r.options.DisableClustering) return results, nil } diff --git a/v2/lib/README.md b/v2/lib/README.md index 3649e54204..fdfc22ce2d 100644 --- a/v2/lib/README.md +++ b/v2/lib/README.md @@ -52,7 +52,7 @@ For Various use cases like batching etc you might want to run nuclei in goroutin wg.Add(1) go func() { defer wg.Done() - err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"})) if err != nil { panic(err) } diff --git a/v2/lib/config.go b/v2/lib/config.go index 89b34cf6d3..fbec5dd7c3 100644 --- a/v2/lib/config.go +++ b/v2/lib/config.go @@ -142,10 +142,9 @@ func EnableHeadlessWithOpts(hopts *HeadlessOpts) NucleiSDKOptions { // StatsOptions type StatsOptions struct { - Interval int - JSON bool - StartMetricServer bool - MetricServerPort int + Interval int + JSON bool + MetricServerPort int } // EnableStats enables Stats collection with defined interval(in sec) and callback @@ -161,7 +160,6 @@ func EnableStatsWithOpts(opts StatsOptions) NucleiSDKOptions { e.opts.StatsInterval = opts.Interval e.enableStats = true e.opts.StatsJSON = opts.JSON - e.opts.Metrics = opts.StartMetricServer e.opts.MetricsPort = opts.MetricServerPort return nil } diff --git a/v2/lib/example_test.go b/v2/lib/example_test.go index 0e15229eb3..e3dd7e8580 100644 --- a/v2/lib/example_test.go +++ b/v2/lib/example_test.go @@ -4,6 +4,9 @@ package nuclei_test import ( + "os" + "testing" + nuclei "github.com/projectdiscovery/nuclei/v2/lib" "github.com/remeh/sizedwaitgroup" ) @@ -12,8 +15,7 @@ import ( func ExampleNucleiEngine() { // create nuclei engine with options ne, err := nuclei.NewNucleiEngine( - nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // only use dns templates - nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"self-signed-ssl"}}), // only run self-signed-ssl template ) if err != nil { panic(err) @@ -28,8 +30,7 @@ func ExampleNucleiEngine() { defer ne.Close() // Output: - // [dns-saas-service-detection] scanme.sh - // [nameserver-fingerprint] scanme.sh + // [self-signed-ssl] scanme.sh:443 } func ExampleThreadSafeNucleiEngine() { @@ -47,8 +48,7 @@ func ExampleThreadSafeNucleiEngine() { go func() { defer sg.Done() err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, - nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), - nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing + nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"nameserver-fingerprint"}}), // only run self-signed-ssl template ) if err != nil { panic(err) @@ -70,7 +70,16 @@ func ExampleThreadSafeNucleiEngine() { defer ne.Close() // Output: - // [dns-saas-service-detection] scanme.sh // [nameserver-fingerprint] scanme.sh // [dns-saas-service-detection] honey.scanme.sh } + +func TestMain(m *testing.M) { + // this file only contains testtables examples https://go.dev/blog/examples + // and actual functionality test are in sdk_test.go + if os.Getenv("GH_ACTION") != "" { + // no need to run this test on github actions + return + } + m.Run() +} diff --git a/v2/lib/sdk_private.go b/v2/lib/sdk_private.go index 5703ed2451..628cdc3b5d 100644 --- a/v2/lib/sdk_private.go +++ b/v2/lib/sdk_private.go @@ -113,11 +113,12 @@ func (e *NucleiEngine) init() error { // setup progressbar if e.enableStats { - progressInstance, progressErr := progress.NewStatsTicker(e.opts.StatsInterval, e.enableStats, e.opts.StatsJSON, e.opts.Metrics, false, e.opts.MetricsPort) + progressInstance, progressErr := progress.NewStatsTicker(e.opts.StatsInterval, e.enableStats, e.opts.StatsJSON, false, e.opts.MetricsPort) if progressErr != nil { return err } e.customProgress = progressInstance + e.interactshOpts.Progress = progressInstance } if err := reporting.CreateConfigIfNotExists(); err != nil { diff --git a/v2/lib/sdk_test.go b/v2/lib/sdk_test.go index 8e36af3be2..021278e747 100644 --- a/v2/lib/sdk_test.go +++ b/v2/lib/sdk_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestExample(t *testing.T) { +func TestSimpleNuclei(t *testing.T) { ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), diff --git a/v2/pkg/core/execute_options.go b/v2/pkg/core/execute_options.go index 4654e7928f..6e890f7b00 100644 --- a/v2/pkg/core/execute_options.go +++ b/v2/pkg/core/execute_options.go @@ -6,6 +6,7 @@ import ( "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -34,13 +35,36 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target results := &atomic.Bool{} selfcontainedWg := &sync.WaitGroup{} + totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count()) + + // attempt to cluster templates if noCluster is false var finalTemplates []*templates.Template + clusterCount := 0 if !noCluster { - finalTemplates, _ = templates.ClusterTemplates(templatesList, e.executerOpts) + finalTemplates, clusterCount = templates.ClusterTemplates(templatesList, e.executerOpts) } else { finalTemplates = templatesList } + totalReqAfterClustering := getRequestCount(finalTemplates) * int(target.Count()) + + if !noCluster && totalReqAfterClustering < totalReqBeforeCluster { + gologger.Info().Msgf("Templates clustered: %d (Reduced %d Requests)", clusterCount, totalReqBeforeCluster-totalReqAfterClustering) + } + + // 0 matches means no templates were found in the directory + if len(finalTemplates) == 0 { + return &atomic.Bool{} + } + + if e.executerOpts.Progress != nil { + // Notes: + // workflow requests are not counted as they can be conditional + // templateList count is user requested templates count (before clustering) + // totalReqAfterClustering is total requests count after clustering + e.executerOpts.Progress.Init(target.Count(), len(templatesList), int64(totalReqAfterClustering)) + } + if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") { // TODO: this is only a placeholder, auto scan strategy should choose scan strategy // based on no of hosts , templates , stream and other optimization parameters @@ -122,3 +146,17 @@ func (e *Engine) executeHostSpray(templatesList []*templates.Template, target In wp.Wait() return results } + +// returns total requests count +func getRequestCount(templates []*templates.Template) int { + count := 0 + for _, template := range templates { + // ignore requests in workflows as total requests in workflow + // depends on what templates will be called in workflow + if len(template.Workflows) > 0 { + continue + } + count += template.TotalRequests + } + return count +} diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go index fc617c57be..5b2db1347f 100644 --- a/v2/pkg/core/workflow_execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -15,7 +15,7 @@ import ( ) func TestWorkflowsSimple(t *testing.T) { - progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ @@ -29,7 +29,7 @@ func TestWorkflowsSimple(t *testing.T) { } func TestWorkflowsSimpleMultiple(t *testing.T) { - progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ @@ -54,7 +54,7 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { } func TestWorkflowsSubtemplates(t *testing.T) { - progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ @@ -80,7 +80,7 @@ func TestWorkflowsSubtemplates(t *testing.T) { } func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { - progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ @@ -104,7 +104,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { } func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { - progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ @@ -133,7 +133,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { } func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { - progressBar, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecutorOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 014be676af..a2d7a49386 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -4,10 +4,7 @@ import ( "context" "encoding/json" "fmt" - "net" - "net/http" "os" - "strconv" "strings" "time" @@ -44,13 +41,12 @@ type StatsTicker struct { cloud bool active bool outputJSON bool - server *http.Server stats clistats.StatisticsClient tickDuration time.Duration } // NewStatsTicker creates and returns a new progress tracking object. -func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port int) (Progress, error) { +func NewStatsTicker(duration int, active, outputJSON, cloud bool, port int) (Progress, error) { var tickDuration time.Duration if active && duration != -1 { tickDuration = time.Duration(duration) * time.Second @@ -60,7 +56,12 @@ func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port progress := &StatsTicker{} - stats, err := clistats.New() + statsOpts := &clistats.DefaultOptions + statsOpts.ListenPort = port + // metrics port is enabled by default and is not configurable with new version of clistats + // by default 63636 is used and than can be modified with -mp flag + + stats, err := clistats.NewWithOptions(context.TODO(), statsOpts) if err != nil { return nil, err } @@ -70,21 +71,6 @@ func NewStatsTicker(duration int, active, outputJSON, metrics, cloud bool, port progress.tickDuration = tickDuration progress.outputJSON = outputJSON - if metrics { - http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { - metrics := progress.getMetrics() - _ = json.NewEncoder(w).Encode(metrics) - }) - progress.server = &http.Server{ - Addr: net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), - Handler: http.DefaultServeMux, - } - go func() { - if err := progress.server.ListenAndServe(); err != nil { - gologger.Warning().Msgf("Could not serve metrics: %s", err) - } - }() - } return progress, nil } @@ -109,13 +95,6 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64) if err := p.stats.Start(); err != nil { gologger.Warning().Msgf("Couldn't start statistics: %s", err) } - - p.stats.GetStatResponse(p.tickDuration, func(s string, err error) error { - if err != nil { - gologger.Warning().Msgf("Could not read statistics: %s\n", err) - } - return nil - }) } } @@ -294,7 +273,4 @@ func (p *StatsTicker) Stop() { gologger.Warning().Msgf("Couldn't stop statistics: %s", err) } } - if p.server != nil { - _ = p.server.Shutdown(context.Background()) - } } diff --git a/v2/pkg/protocols/multi/request_test.go b/v2/pkg/protocols/multi/request_test.go index 660f69afe8..829f54a8c3 100644 --- a/v2/pkg/protocols/multi/request_test.go +++ b/v2/pkg/protocols/multi/request_test.go @@ -23,7 +23,7 @@ var executerOpts protocols.ExecutorOptions func setup() { options := testutils.DefaultOptions testutils.Init(options) - progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(), diff --git a/v2/pkg/templates/compile_test.go b/v2/pkg/templates/compile_test.go index 091dc07a9f..2b73005111 100644 --- a/v2/pkg/templates/compile_test.go +++ b/v2/pkg/templates/compile_test.go @@ -37,7 +37,7 @@ var executerOpts protocols.ExecutorOptions func setup() { options := testutils.DefaultOptions testutils.Init(options) - progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(), diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 7f0b16a377..83fee7b202 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -78,7 +78,7 @@ type TemplateInfo struct { // NewMockExecuterOptions creates a new mock executeroptions struct func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions { - progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts := &protocols.ExecutorOptions{ TemplateID: info.ID, TemplateInfo: info.Info, diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 9aef497cf3..3675fd889a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -210,7 +210,7 @@ type Options struct { SystemResolvers bool // ShowActions displays a list of all headless actions ShowActions bool - // Metrics enables display of metrics via an http endpoint + // Deprecated: Enabled by default through clistats . Metrics enables display of metrics via an http endpoint Metrics bool // Debug mode allows debugging request/responses for the engine Debug bool From a093f0f2337b6c65f2e4f9b4a6e8acbacdfc3615 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 19:04:26 +0530 Subject: [PATCH 05/10] fix lint error --- v2/pkg/progress/progress.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index a2d7a49386..b29e4894a4 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -244,11 +244,6 @@ func metricsMap(stats clistats.StatisticsClient) map[string]interface{} { return results } -// getMetrics returns a map of important metrics for client -func (p *StatsTicker) getMetrics() map[string]interface{} { - return metricsMap(p.stats) -} - // fmtDuration formats the duration for the time elapsed func fmtDuration(d time.Duration) string { d = d.Round(time.Second) From 040a44ba8a1c70105a74ccddb0a887376d2ddcad Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 19:08:13 +0530 Subject: [PATCH 06/10] fix missing ticker --- v2/pkg/progress/progress.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index b29e4894a4..d4c9e3c509 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -95,6 +95,14 @@ func (p *StatsTicker) Init(hostCount int64, rulesCount int, requestCount int64) if err := p.stats.Start(); err != nil { gologger.Warning().Msgf("Couldn't start statistics: %s", err) } + + // Note: this is needed and is responsible for the tick event + p.stats.GetStatResponse(p.tickDuration, func(s string, err error) error { + if err != nil { + gologger.Warning().Msgf("Could not read statistics: %s\n", err) + } + return nil + }) } } From 186a4dafa2e98d0182641243edcb65a8c2a3679e Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 19:14:18 +0530 Subject: [PATCH 07/10] update advanced library usage example --- v2/examples/advanced/advanced.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/v2/examples/advanced/advanced.go b/v2/examples/advanced/advanced.go index 25dcd00985..960722546d 100644 --- a/v2/examples/advanced/advanced.go +++ b/v2/examples/advanced/advanced.go @@ -21,18 +21,17 @@ func main() { defer sg.Done() err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), - nuclei.WithConcurrency(nuclei.Concurrency{TemplateConcurrency: 1}), // never use templateconcurrency 1. this is just for testing ) if err != nil { panic(err) } }() - // scan 2 = run dns templates on honey.scanme.sh + // scan 2 = run templates with oast tags on honey.scanme.sh sg.Add() go func() { defer sg.Done() - err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + err = ne.ExecuteNucleiWithOpts([]string{"http://honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}})) if err != nil { panic(err) } From 535fb6f98581ef435ab08b58fcdb4abb7858d55e Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 30 Aug 2023 19:49:16 +0530 Subject: [PATCH 08/10] fix integration tests --- v2/cmd/integration-test/template-path.go | 2 +- v2/internal/runner/runner.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 773f919254..c5943aeffd 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{ //template folder path issue {Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd - {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, + {Path: "./dns/detect-dangling-cname.yaml", TestCase: &cwdTemplateTest{}}, //relative path {Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index a742bd9cf4..49364b0f84 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -675,6 +675,10 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) finalTemplates = append(finalTemplates, store.Templates()...) finalTemplates = append(finalTemplates, store.Workflows()...) + if len(finalTemplates) == 0 { + return nil, errors.New("no templates provided for scan") + } + results := engine.ExecuteScanWithOpts(finalTemplates, r.hmapInputProvider, r.options.DisableClustering) return results, nil } From 2ba5402e7259f5a2e443859f46df2b1f41765bff Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 1 Sep 2023 23:38:19 +0530 Subject: [PATCH 09/10] misc update --- v2/examples/advanced/advanced.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/examples/advanced/advanced.go b/v2/examples/advanced/advanced.go index 960722546d..86a871a22d 100644 --- a/v2/examples/advanced/advanced.go +++ b/v2/examples/advanced/advanced.go @@ -12,8 +12,7 @@ func main() { panic(err) } // setup sizedWaitgroup to handle concurrency - // here we are using sizedWaitgroup to limit concurrency to 1 - sg := sizedwaitgroup.New(1) + sg := sizedwaitgroup.New(10) // scan 1 = run dns templates on scanme.sh sg.Add() From b8faecc2630c60b28806ebfbda2594078df39e3f Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Sat, 2 Sep 2023 14:13:14 +0530 Subject: [PATCH 10/10] add utm_source and fix lint error --- v2/internal/installer/util.go | 26 ++++++++++++++++++++++ v2/internal/installer/versioncheck.go | 1 + v2/pkg/tmplexec/flow/flow_executor_test.go | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/v2/internal/installer/util.go b/v2/internal/installer/util.go index 024202a340..57ae05bb26 100644 --- a/v2/internal/installer/util.go +++ b/v2/internal/installer/util.go @@ -103,3 +103,29 @@ func isEmptyDir(dir string) bool { }) return !hasFiles } + +// getUtmSource returns utm_source from environment variable +func getUtmSource() string { + value := "" + switch { + case os.Getenv("GH_ACTION") != "": + value = "ghci" + case os.Getenv("TRAVIS") != "": + value = "travis" + case os.Getenv("CIRCLECI") != "": + value = "circleci" + case os.Getenv("CI") != "": + value = "gitlabci" // this also includes bitbucket + case os.Getenv("GITHUB_ACTIONS") != "": + value = "ghci" + case os.Getenv("AWS_EXECUTION_ENV") != "": + value = os.Getenv("AWS_EXECUTION_ENV") + case os.Getenv("JENKINS_URL") != "": + value = "jenkins" + case os.Getenv("FUNCTION_TARGET") != "": + value = "gcf" + default: + value = "unknown" + } + return value +} diff --git a/v2/internal/installer/versioncheck.go b/v2/internal/installer/versioncheck.go index 88857410da..fa92052497 100644 --- a/v2/internal/installer/versioncheck.go +++ b/v2/internal/installer/versioncheck.go @@ -63,6 +63,7 @@ func getpdtmParams(isSDK bool) string { if isSDK { params.Add("sdk", "true") } + params.Add("utm_source", getUtmSource()) return params.Encode() } diff --git a/v2/pkg/tmplexec/flow/flow_executor_test.go b/v2/pkg/tmplexec/flow/flow_executor_test.go index 7c7ec983c5..5680735b7e 100644 --- a/v2/pkg/tmplexec/flow/flow_executor_test.go +++ b/v2/pkg/tmplexec/flow/flow_executor_test.go @@ -23,7 +23,7 @@ var executerOpts protocols.ExecutorOptions func setup() { options := testutils.DefaultOptions testutils.Init(options) - progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(),