From a117c685719172e3be2e3e4af524559b7dd27f0d Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jun 2024 11:45:48 +0200 Subject: [PATCH 1/9] globals => shared struct --- lib/sdk.go | 3 ++ lib/sdk_private.go | 9 ++++ pkg/protocols/common/protocolinit/init.go | 8 ---- pkg/protocols/common/protocolstate/state.go | 40 +++++++++++------- pkg/protocols/network/network.go | 8 +--- .../network/networkclientpool/clientpool.go | 34 --------------- pkg/protocols/protocols.go | 42 +++++++++++++++++++ pkg/protocols/ssl/ssl.go | 9 +--- pkg/protocols/websocket/websocket.go | 8 +--- .../whois/rdapclientpool/clientpool.go | 38 ----------------- pkg/protocols/whois/whois.go | 5 ++- 11 files changed, 88 insertions(+), 116 deletions(-) delete mode 100644 pkg/protocols/network/networkclientpool/clientpool.go delete mode 100644 pkg/protocols/whois/rdapclientpool/clientpool.go diff --git a/lib/sdk.go b/lib/sdk.go index 60f5255bcd..4661f0ccbb 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -214,6 +214,9 @@ func (e *NucleiEngine) closeInternal() { if e.httpxClient != nil { _ = e.httpxClient.Close() } + if e.executerOpts.Dialers != nil { + e.executerOpts.Dialers.Close() + } } // Close all resources used by nuclei engine diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 3475200bd0..f7bac19592 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -158,6 +158,14 @@ func (e *NucleiEngine) init(ctx context.Context) error { e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) } + dialer, err := protocolstate.GetDialerFromOptions(e.opts) + if err != nil { + return errors.Wrap(err, "could not create dialer") + } + + dialers := &protocols.Dialers{} + dialers.SetDefault(dialer) + e.executerOpts = protocols.ExecutorOptions{ Output: e.customWriter, Options: e.opts, @@ -171,6 +179,7 @@ func (e *NucleiEngine) init(ctx context.Context) error { ResumeCfg: types.NewResumeCfg(), Browser: e.browserInstance, Parser: e.parser, + Dialers: dialers, } if len(e.opts.SecretsFile) > 0 { authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts) diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go index c8268337f5..c02d46309a 100644 --- a/pkg/protocols/common/protocolinit/init.go +++ b/pkg/protocols/common/protocolinit/init.go @@ -6,8 +6,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/types" ) @@ -25,12 +23,6 @@ func Init(options *types.Options) error { if err := signerpool.Init(options); err != nil { return err } - if err := networkclientpool.Init(options); err != nil { - return err - } - if err := rdapclientpool.Init(options); err != nil { - return err - } if err := compiler.Init(options); err != nil { return err } diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index cf6f5234ec..07eb93e75a 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -26,12 +26,7 @@ func ShouldInit() bool { return Dialer == nil } -// Init creates the Dialer instance based on user configuration -func Init(options *types.Options) error { - if Dialer != nil { - return nil - } - +func GetDialerFromOptions(options *types.Options) (*fastdialer.Dialer, error) { lfaAllowed = options.AllowLocalFileAccess opts := fastdialer.DefaultOptions if options.DialerTimeout > 0 { @@ -66,7 +61,7 @@ func Init(options *types.Options) error { case options.SourceIP != "" && options.Interface != "": isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, options.Interface) if err != nil { - return err + return nil, err } if isAssociated { opts.Dialer = &net.Dialer{ @@ -75,12 +70,12 @@ func Init(options *types.Options) error { }, } } else { - return fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface) + return nil, fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface) } case options.SourceIP != "": isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, "any") if err != nil { - return err + return nil, err } if isAssociated { opts.Dialer = &net.Dialer{ @@ -89,12 +84,12 @@ func Init(options *types.Options) error { }, } } else { - return fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP) + return nil, fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP) } case options.Interface != "": ifadrr, err := interfaceAddress(options.Interface) if err != nil { - return err + return nil, err } opts.Dialer = &net.Dialer{ LocalAddr: &net.TCPAddr{ @@ -105,7 +100,7 @@ func Init(options *types.Options) error { if types.ProxySocksURL != "" { proxyURL, err := url.Parse(types.ProxySocksURL) if err != nil { - return err + return nil, err } var forward *net.Dialer if opts.Dialer != nil { @@ -119,7 +114,7 @@ func Init(options *types.Options) error { } dialer, err := proxy.FromURL(proxyURL, forward) if err != nil { - return err + return nil, err } opts.ProxyDialer = &dialer } @@ -143,9 +138,26 @@ func Init(options *types.Options) error { // fastdialer now by default fallbacks to ztls when there are tls related errors dialer, err := fastdialer.NewDialer(opts) if err != nil { - return errors.Wrap(err, "could not create dialer") + return nil, errors.Wrap(err, "could not create dialer") + } + + return dialer, nil +} + +// Init creates the Dialer instance based on user configuration +func Init(options *types.Options) error { + // *************************** + // todo: remove this part + if Dialer != nil { + return nil + } + + dialer, err := GetDialerFromOptions(options) + if err != nil { + return err } Dialer = dialer + // ********************+***** // override dialer in mysql mysql.RegisterDialContext("tcp", func(ctx context.Context, addr string) (net.Conn, error) { diff --git a/pkg/protocols/network/network.go b/pkg/protocols/network/network.go index 70d618dcb5..23412af799 100644 --- a/pkg/protocols/network/network.go +++ b/pkg/protocols/network/network.go @@ -11,7 +11,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -232,12 +231,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads) } - // Create a client for the class - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errors.Wrap(err, "could not get network client") - } - request.dialer = client + request.dialer = options.Dialers.Default() if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators diff --git a/pkg/protocols/network/networkclientpool/clientpool.go b/pkg/protocols/network/networkclientpool/clientpool.go deleted file mode 100644 index a67cee2967..0000000000 --- a/pkg/protocols/network/networkclientpool/clientpool.go +++ /dev/null @@ -1,34 +0,0 @@ -package networkclientpool - -import ( - "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v3/pkg/types" -) - -var ( - normalClient *fastdialer.Dialer -) - -// Init initializes the clientpool implementation -func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } - normalClient = protocolstate.Dialer - return nil -} - -// Configuration contains the custom configuration options for a client -type Configuration struct{} - -// Hash returns the hash of the configuration to allow client pooling -func (c *Configuration) Hash() string { - return "" -} - -// Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration /*TODO review unused parameters*/) (*fastdialer.Dialer, error) { - return normalClient, nil -} diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index af7d2b4766..82585e21fe 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -3,9 +3,13 @@ package protocols import ( "context" "encoding/base64" + "sync" "sync/atomic" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/rdap" mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" @@ -124,6 +128,44 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool + // shared dialers + Dialers *Dialers +} + +type Dialers struct { + fastDialer *fastdialer.Dialer + + rdapClient *rdap.Client + rdapOnce sync.Once +} + +func (d *Dialers) SetDefault(fastDialer *fastdialer.Dialer) { + d.fastDialer = fastDialer +} + +func (d *Dialers) Default() *fastdialer.Dialer { + return d.fastDialer +} + +func (d *Dialers) Rdap(withDebug bool) *rdap.Client { + d.rdapOnce.Do(func() { + d.rdapClient = &rdap.Client{} + if withDebug { + d.rdapClient.Verbose = func(text string) { + gologger.Debug().Msgf("rdap: %s", text) + } + } + }) + return d.rdapClient +} + +func (d *Dialers) Close() { + if d.fastDialer != nil { + d.fastDialer.Close() + } + if d.rdapClient != nil { + d.rdapClient.HTTP.CloseIdleConnections() + } } // todo: centralizing components is not feasible with current clogged architecture diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 49d612b882..3517cf2f4b 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -23,7 +23,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -113,12 +112,8 @@ func (request *Request) CanCluster(other *Request) bool { // Compile compiles the request generators preparing any requests possible. func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.options = options + request.dialer = options.Dialers.Default() - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errorutil.NewWithTag("ssl", "could not get network client").Wrap(err) - } - request.dialer = client switch { //validate scanmode case request.ScanMode == "": @@ -153,7 +148,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { WildcardCertCheck: true, Retries: request.options.Options.Retries, Timeout: request.options.Options.Timeout, - Fastdialer: client, + Fastdialer: options.Dialers.Default(), ClientHello: true, ServerHello: true, DisplayDns: true, diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index a956de0e3f..9144b9e04b 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -28,7 +28,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -100,13 +99,10 @@ const ( func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.options = options - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errors.Wrap(err, "could not get network client") - } - request.dialer = client + request.dialer = options.Dialers.Default() if len(request.Payloads) > 0 { + var err error request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions()) if err != nil { return errors.Wrap(err, "could not parse payloads") diff --git a/pkg/protocols/whois/rdapclientpool/clientpool.go b/pkg/protocols/whois/rdapclientpool/clientpool.go deleted file mode 100644 index cb393a5055..0000000000 --- a/pkg/protocols/whois/rdapclientpool/clientpool.go +++ /dev/null @@ -1,38 +0,0 @@ -package rdapclientpool - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/types" - "github.com/projectdiscovery/rdap" -) - -var normalClient *rdap.Client - -// Init initializes the client pool implementation -func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } - - normalClient = &rdap.Client{} - if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse { - normalClient.Verbose = func(text string) { - gologger.Debug().Msgf("rdap: %s", text) - } - } - return nil -} - -// Configuration contains the custom configuration options for a client - placeholder -type Configuration struct{} - -// Hash returns the hash of the configuration to allow client pooling - placeholder -func (c *Configuration) Hash() string { - return "" -} - -// Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) { - return normalClient, nil -} diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index ce0bafe56c..9d0e19f94e 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -22,7 +22,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -64,7 +63,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.options = options - request.client, _ = rdapclientpool.Get(options.Options, nil) + + hasDebug := options.Options.Verbose || options.Options.Debug || options.Options.DebugRequests || options.Options.DebugResponse + request.client = options.Dialers.Rdap(hasDebug) if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators From a976407625cfdfbefe5241a631dec3946cad6bbb Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jun 2024 12:59:33 +0200 Subject: [PATCH 2/9] tmp init --- cmd/integration-test/library.go | 7 ++++++ internal/runner/runner.go | 6 +++++ lib/sdk_private.go | 14 ++++------- pkg/protocols/protocols.go | 35 +++++++++++++++------------ pkg/protocols/whois/whois.go | 3 +-- pkg/tmplexec/multiproto/multi_test.go | 17 +++++++------ pkg/types/types.go | 4 +++ 7 files changed, 53 insertions(+), 33 deletions(-) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index 1324688e05..4aba74a040 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -100,6 +100,12 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) ratelimiter := ratelimit.New(context.Background(), 150, time.Second) defer ratelimiter.Stop() + dialers, err := protocols.NewDealers(defaultOpts) + if err != nil { + return nil, errors.Wrap(err, "could not create dialers") + } + defer dialers.Close() + executerOpts := protocols.ExecutorOptions{ Output: outputWriter, Options: defaultOpts, @@ -112,6 +118,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: templates.NewParser(), + Dialers: dialers, } engine := core.New(defaultOpts) engine.SetExecuterOptions(executerOpts) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index bc436500af..6f82acfc5c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -453,6 +453,11 @@ func (r *Runner) RunEnumeration() error { fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency) r.fuzzFrequencyCache = fuzzFreqCache + dialers, err := protocols.NewDealers(r.options) + if err != nil { + return errors.Wrap(err, "could not create dialers") + } + // Create the executor options which will be used throughout the execution // stage by the nuclei engine modules. executorOpts := protocols.ExecutorOptions{ @@ -472,6 +477,7 @@ func (r *Runner) RunEnumeration() error { TemporaryDirectory: r.tmpDir, Parser: r.parser, FuzzParamsFrequency: fuzzFreqCache, + Dialers: dialers, } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { diff --git a/lib/sdk_private.go b/lib/sdk_private.go index f7bac19592..0f0115f8e9 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -108,6 +108,11 @@ func (e *NucleiEngine) init(ctx context.Context) error { return err } + dialers, err := protocols.NewDealers(e.opts) + if err != nil { + return errors.Wrap(err, "could not create dialers") + } + if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) if err != nil { @@ -127,7 +132,6 @@ func (e *NucleiEngine) init(ctx context.Context) error { }) e.applyRequiredDefaults(ctx) - var err error // setup progressbar if e.enableStats { @@ -158,14 +162,6 @@ func (e *NucleiEngine) init(ctx context.Context) error { e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) } - dialer, err := protocolstate.GetDialerFromOptions(e.opts) - if err != nil { - return errors.Wrap(err, "could not create dialer") - } - - dialers := &protocols.Dialers{} - dialers.SetDefault(dialer) - e.executerOpts = protocols.ExecutorOptions{ Output: e.customWriter, Options: e.opts, diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 82585e21fe..fbf300f450 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -3,7 +3,6 @@ package protocols import ( "context" "encoding/base64" - "sync" "sync/atomic" "github.com/projectdiscovery/fastdialer/fastdialer" @@ -31,6 +30,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" @@ -134,28 +134,33 @@ type ExecutorOptions struct { type Dialers struct { fastDialer *fastdialer.Dialer - rdapClient *rdap.Client - rdapOnce sync.Once } -func (d *Dialers) SetDefault(fastDialer *fastdialer.Dialer) { - d.fastDialer = fastDialer +func NewDealers(options *types.Options) (*Dialers, error) { + var dialers = &Dialers{} + var err error + + dialers.fastDialer, err = protocolstate.GetDialerFromOptions(options) + if err != nil { + return nil, err + } + + dialers.rdapClient = &rdap.Client{} + if options.HasDebug() { + dialers.rdapClient.Verbose = func(text string) { + gologger.Debug().Msgf("rdap: %s", text) + } + } + + return dialers, nil } func (d *Dialers) Default() *fastdialer.Dialer { return d.fastDialer } -func (d *Dialers) Rdap(withDebug bool) *rdap.Client { - d.rdapOnce.Do(func() { - d.rdapClient = &rdap.Client{} - if withDebug { - d.rdapClient.Verbose = func(text string) { - gologger.Debug().Msgf("rdap: %s", text) - } - } - }) +func (d *Dialers) Rdap() *rdap.Client { return d.rdapClient } @@ -163,7 +168,7 @@ func (d *Dialers) Close() { if d.fastDialer != nil { d.fastDialer.Close() } - if d.rdapClient != nil { + if d.rdapClient.HTTP != nil { d.rdapClient.HTTP.CloseIdleConnections() } } diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index 9d0e19f94e..cf3b9d8061 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -64,8 +64,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.options = options - hasDebug := options.Options.Verbose || options.Options.Debug || options.Options.DebugRequests || options.Options.DebugResponse - request.client = options.Dialers.Rdap(hasDebug) + request.client = options.Dialers.Rdap() if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index 4f2aa25e4e..cf90cc850b 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -2,7 +2,6 @@ package multiproto_test import ( "context" - "log" "testing" "time" @@ -21,11 +20,14 @@ import ( var executerOpts protocols.ExecutorOptions -func setup() { +func setup(t *testing.T) { options := testutils.DefaultOptions testutils.Init(options) progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) + dialers, err := protocols.NewDealers(options) + require.Nil(t, err, "could not create dialers") + executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, @@ -36,16 +38,16 @@ func setup() { Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), Parser: templates.NewParser(), + Dialers: dialers, } workflowLoader, err := workflow.NewLoader(&executerOpts) - if err != nil { - log.Fatalf("Could not create workflow loader: %s\n", err) - } + require.Nil(t, err, "could not create workflow loader") executerOpts.WorkflowLoader = workflowLoader } func TestMultiProtoWithDynamicExtractor(t *testing.T) { - setup() + setup(t) + Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") @@ -62,7 +64,8 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) { } func TestMultiProtoWithProtoPrefix(t *testing.T) { - setup() + setup(t) + Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") diff --git a/pkg/types/types.go b/pkg/types/types.go index f49da194cc..b9ef397524 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -428,6 +428,10 @@ func (options *Options) HasClientCertificates() bool { return options.ClientCertFile != "" || options.ClientCAFile != "" || options.ClientKeyFile != "" } +func (options *Options) HasDebug() bool { + return options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse +} + // DefaultOptions returns default options for nuclei func DefaultOptions() *Options { return &Options{ From e23e28477edf11f32f7fe2e9918af1438e4bade5 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jun 2024 13:56:10 +0200 Subject: [PATCH 3/9] fixing tests --- internal/runner/options.go | 3 +- pkg/js/compiler/compiler.go | 5 --- pkg/js/compiler/pool.go | 5 --- pkg/protocols/code/code_test.go | 9 ++-- pkg/protocols/dns/dns_test.go | 27 ++++++++---- pkg/protocols/dns/operators_test.go | 33 +++++++++------ pkg/protocols/dns/request_test.go | 8 ++-- pkg/protocols/file/find_test.go | 8 ++-- pkg/protocols/file/operators_test.go | 33 +++++++++------ pkg/protocols/file/request_test.go | 10 +++-- pkg/protocols/headless/engine/page_actions.go | 7 ---- pkg/protocols/http/build_request_test.go | 41 ++++++++++++------- pkg/protocols/http/http_test.go | 8 ++-- pkg/protocols/http/operators_test.go | 32 +++++++++------ pkg/protocols/http/request_test.go | 26 +++++++----- pkg/protocols/network/network_test.go | 9 ++-- pkg/protocols/network/operators_test.go | 37 +++++++++++------ pkg/protocols/network/request_test.go | 7 +++- pkg/protocols/offlinehttp/find_test.go | 8 ++-- pkg/protocols/offlinehttp/operators_test.go | 30 ++++++++------ pkg/protocols/ssl/ssl_test.go | 8 ++-- pkg/templates/cluster_test.go | 6 ++- pkg/testutils/fuzzplayground/server.go | 1 - pkg/testutils/testutils.go | 9 +++- pkg/tmplexec/flow/flow_executor.go | 5 --- 25 files changed, 225 insertions(+), 150 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index 8798c73113..ebb49fff7e 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -32,7 +32,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" fileutil "github.com/projectdiscovery/utils/file" "github.com/projectdiscovery/utils/generic" - logutil "github.com/projectdiscovery/utils/log" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -342,7 +341,7 @@ func configureOutput(options *types.Options) { } // disable standard logger (ref: https://github.com/golang/go/issues/19895) - logutil.DisableDefaultLogger() + // logutil.DisableDefaultLogger() } // loadResolvers loads resolvers from both user-provided flags and file diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 367bd013b2..825dd7eea3 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -117,11 +117,6 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() return ExecuteProgram(program, args, opts) }) if err != nil { diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go index 17d9f746f5..53943efcb7 100644 --- a/pkg/js/compiler/pool.go +++ b/pkg/js/compiler/pool.go @@ -84,11 +84,6 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg opts.Cleanup(runtime) } }() - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %s", r) - } - }() // set template ctx _ = runtime.Set("template", args.TemplateCtx) // set args diff --git a/pkg/protocols/code/code_test.go b/pkg/protocols/code/code_test.go index 320f7c5481..29bc0af00c 100644 --- a/pkg/protocols/code/code_test.go +++ b/pkg/protocols/code/code_test.go @@ -24,11 +24,14 @@ func TestCodeProtocol(t *testing.T) { Engine: []string{"sh"}, Source: "echo test", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile code request") var gotEvent output.InternalEvent diff --git a/pkg/protocols/dns/dns_test.go b/pkg/protocols/dns/dns_test.go index bf2c3a9f76..392c13e0c5 100644 --- a/pkg/protocols/dns/dns_test.go +++ b/pkg/protocols/dns/dns_test.go @@ -24,11 +24,13 @@ func TestDNSCompileMake(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req, err := request.Make("one.one.one.one", map[string]interface{}{"FQDN": "one.one.one.one"}) @@ -53,11 +55,14 @@ func TestDNSRequests(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") reqCount := request.Requests() @@ -76,11 +81,15 @@ func TestDNSRequests(t *testing.T) { Name: "{{subdomain}}.{{FQDN}}", Payloads: map[string]interface{}{"subdomain": []string{"a", "b", "c"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") reqCount := request.Requests() diff --git a/pkg/protocols/dns/operators_test.go b/pkg/protocols/dns/operators_test.go index 80d02908de..59a9e688ec 100644 --- a/pkg/protocols/dns/operators_test.go +++ b/pkg/protocols/dns/operators_test.go @@ -31,11 +31,13 @@ func TestResponseToDSLMap(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) @@ -65,11 +67,13 @@ func TestDNSOperatorMatch(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) @@ -177,11 +181,14 @@ func TestDNSOperatorExtract(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) @@ -227,10 +234,12 @@ func TestDNSMakeResult(t *testing.T) { recursion := false testutils.Init(options) templateID := "testing-dns" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") request := &Request{ RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", @@ -253,7 +262,7 @@ func TestDNSMakeResult(t *testing.T) { }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) diff --git a/pkg/protocols/dns/request_test.go b/pkg/protocols/dns/request_test.go index 81f0b98ab2..a850a04351 100644 --- a/pkg/protocols/dns/request_test.go +++ b/pkg/protocols/dns/request_test.go @@ -22,10 +22,12 @@ func TestDNSExecuteWithResults(t *testing.T) { recursion := false testutils.Init(options) templateID := "testing-dns" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") request := &Request{ RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", @@ -48,7 +50,7 @@ func TestDNSExecuteWithResults(t *testing.T) { }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") var finalEvent *output.InternalWrappedEvent diff --git a/pkg/protocols/file/find_test.go b/pkg/protocols/file/find_test.go index 3df5d2383b..ea692c1410 100644 --- a/pkg/protocols/file/find_test.go +++ b/pkg/protocols/file/find_test.go @@ -26,11 +26,13 @@ func TestFindInputPaths(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") tempDir, err := os.MkdirTemp("", "test-*") diff --git a/pkg/protocols/file/operators_test.go b/pkg/protocols/file/operators_test.go index 72a0ca45bd..d874533488 100644 --- a/pkg/protocols/file/operators_test.go +++ b/pkg/protocols/file/operators_test.go @@ -40,11 +40,13 @@ func TestResponseToDSLMap(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := "test-data\r\n" @@ -66,11 +68,13 @@ func TestFileOperatorMatch(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := "test-data\r\n1.1.1.1\r\n" @@ -155,11 +159,13 @@ func TestFileOperatorExtract(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := "test-data\r\n1.1.1.1\r\n" @@ -255,10 +261,13 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi testutils.Init(options) templateID := "testing-file" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + request := &Request{ ID: templateID, MaxSize: "1Gb", @@ -276,7 +285,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") matchedFileName := "test.txt" diff --git a/pkg/protocols/file/request_test.go b/pkg/protocols/file/request_test.go index 7c69cf5bce..a180a30c09 100644 --- a/pkg/protocols/file/request_test.go +++ b/pkg/protocols/file/request_test.go @@ -24,10 +24,14 @@ func TestFileExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-file" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + request := &Request{ ID: templateID, MaxSize: "1Gb", @@ -49,7 +53,7 @@ func TestFileExecuteWithResults(t *testing.T) { }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") tempDir, err := os.MkdirTemp("", "test-*") diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 57cc371093..c7285eccc1 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -53,13 +53,6 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var // typically used for waitEvent waitFuncs := make([]func() error, 0) - // avoid any future panics caused due to go-rod library - defer func() { - if r := recover(); r != nil { - err = errorutil.New("panic on headless action: %v", r) - } - }() - for _, act := range actions { switch act.ActionType.ActionType { case ActionNavigate: diff --git a/pkg/protocols/http/build_request_test.go b/pkg/protocols/http/build_request_test.go index 4405bd10b7..700603bdd6 100644 --- a/pkg/protocols/http/build_request_test.go +++ b/pkg/protocols/http/build_request_test.go @@ -31,11 +31,14 @@ func TestMakeRequestFromModal(t *testing.T) { "Content-Length": "1", }, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -61,11 +64,13 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { Path: []string{"{{BaseURL}}?query=example"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -101,11 +106,13 @@ Connection: close Authorization: Basic {{username + ':' + password}} Accept-Encoding: gzip`}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -142,11 +149,13 @@ Connection: close Authorization: Basic {{base64(username + ':' + password)}} Accept-Encoding: gzip`}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -175,11 +184,13 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) diff --git a/pkg/protocols/http/http_test.go b/pkg/protocols/http/http_test.go index 396fd55eea..b97c009126 100644 --- a/pkg/protocols/http/http_test.go +++ b/pkg/protocols/http/http_test.go @@ -31,11 +31,13 @@ Connection: close Authorization: Basic {{username + ':' + password}} Accept-Encoding: gzip`}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") require.Equal(t, 6, request.Requests(), "could not get correct number of requests") require.Equal(t, map[string]string{"User-Agent": "test", "Hello": "World"}, request.customHeaders, "could not get correct custom headers") diff --git a/pkg/protocols/http/operators_test.go b/pkg/protocols/http/operators_test.go index deb5319d9b..832a15c533 100644 --- a/pkg/protocols/http/operators_test.go +++ b/pkg/protocols/http/operators_test.go @@ -27,11 +27,13 @@ func TestResponseToDSLMap(t *testing.T) { Path: []string{"{{BaseURL}}?test=1"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -57,11 +59,13 @@ func TestHTTPOperatorMatch(t *testing.T) { Path: []string{"{{BaseURL}}?test=1"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -145,11 +149,13 @@ func TestHTTPOperatorExtract(t *testing.T) { Path: []string{"{{BaseURL}}?test=1"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -272,11 +278,13 @@ func TestHTTPMakeResult(t *testing.T) { }}, }, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} diff --git a/pkg/protocols/http/request_test.go b/pkg/protocols/http/request_test.go index c0bd2bb346..76b3a4806b 100644 --- a/pkg/protocols/http/request_test.go +++ b/pkg/protocols/http/request_test.go @@ -71,12 +71,13 @@ Disallow: /c`)) })) defer ts.Close() - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") var finalEvent *output.InternalWrappedEvent @@ -144,12 +145,13 @@ func TestDisableTE(t *testing.T) { })) defer ts.Close() - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http raw request") err = request2.Compile(executerOpts) @@ -220,10 +222,12 @@ func TestReqURLPattern(t *testing.T) { })) defer ts.Close() - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") client, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress)) executerOpts.Interactsh = client defer client.Close() @@ -235,7 +239,7 @@ func TestReqURLPattern(t *testing.T) { "{{randstr}}": "2eNU2kbrOcUDzhnUL1RGvSo1it7", } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") var finalEvent *output.InternalWrappedEvent diff --git a/pkg/protocols/network/network_test.go b/pkg/protocols/network/network_test.go index 83dbc7fc6d..f3a30db804 100644 --- a/pkg/protocols/network/network_test.go +++ b/pkg/protocols/network/network_test.go @@ -21,11 +21,14 @@ func TestNetworkCompileMake(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") require.Equal(t, 1, len(request.addresses), "could not get correct number of input address") diff --git a/pkg/protocols/network/operators_test.go b/pkg/protocols/network/operators_test.go index 88bc94b4d6..112997081f 100644 --- a/pkg/protocols/network/operators_test.go +++ b/pkg/protocols/network/operators_test.go @@ -25,11 +25,14 @@ func TestResponseToDSLMap(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data\r\n"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" @@ -50,11 +53,15 @@ func TestNetworkOperatorMatch(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data\r\n"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" @@ -135,11 +142,14 @@ func TestNetworkOperatorExtract(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data\r\n"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" @@ -198,11 +208,14 @@ func TestNetworkMakeResult(t *testing.T) { }}, }, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" diff --git a/pkg/protocols/network/request_test.go b/pkg/protocols/network/request_test.go index 1945888e9b..9e2dc06627 100644 --- a/pkg/protocols/network/request_test.go +++ b/pkg/protocols/network/request_test.go @@ -55,10 +55,13 @@ func TestNetworkExecuteWithResults(t *testing.T) { request.Address[0] = "{{Hostname}}" request.Inputs = append(request.Inputs, &Input{Data: fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host)}) - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") diff --git a/pkg/protocols/offlinehttp/find_test.go b/pkg/protocols/offlinehttp/find_test.go index 83249bc97f..08d1d3e31d 100644 --- a/pkg/protocols/offlinehttp/find_test.go +++ b/pkg/protocols/offlinehttp/find_test.go @@ -20,12 +20,14 @@ func TestFindResponses(t *testing.T) { testutils.Init(options) templateID := "testing-offline" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") tempDir, err := os.MkdirTemp("", "test-*") diff --git a/pkg/protocols/offlinehttp/operators_test.go b/pkg/protocols/offlinehttp/operators_test.go index 21bb869392..546d988b2c 100644 --- a/pkg/protocols/offlinehttp/operators_test.go +++ b/pkg/protocols/offlinehttp/operators_test.go @@ -22,12 +22,14 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -48,12 +50,13 @@ func TestHTTPOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -117,12 +120,13 @@ func TestHTTPOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -171,10 +175,12 @@ func TestHTTPMakeResult(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") executerOpts.Operators = []*operators.Operators{{ Matchers: []*matchers.Matcher{{ Name: "test", @@ -188,7 +194,7 @@ func TestHTTPMakeResult(t *testing.T) { Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} diff --git a/pkg/protocols/ssl/ssl_test.go b/pkg/protocols/ssl/ssl_test.go index 59c9f85f3e..f57f1d9a40 100644 --- a/pkg/protocols/ssl/ssl_test.go +++ b/pkg/protocols/ssl/ssl_test.go @@ -21,11 +21,13 @@ func TestSSLProtocol(t *testing.T) { request := &Request{ Address: "{{Hostname}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile ssl request") var gotEvent output.InternalEvent diff --git a/pkg/templates/cluster_test.go b/pkg/templates/cluster_test.go index c8e68b3e07..c6fa356ff2 100644 --- a/pkg/templates/cluster_test.go +++ b/pkg/templates/cluster_test.go @@ -13,10 +13,12 @@ import ( func TestClusterTemplates(t *testing.T) { // state of whether template is flow or multiprotocol is stored in executerOptions i.e why we need to pass it - execOptions := testutils.NewMockExecuterOptions(testutils.DefaultOptions, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: "templateID", Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + execOptions, err := testutils.NewMockExecuterOptions(testutils.DefaultOptions, templateInfo) + require.Nil(t, err, "could not create executer options") t.Run("http-cluster-get", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} tp2 := &Template{Path: "second.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} diff --git a/pkg/testutils/fuzzplayground/server.go b/pkg/testutils/fuzzplayground/server.go index af42552cbf..f0029fcb15 100644 --- a/pkg/testutils/fuzzplayground/server.go +++ b/pkg/testutils/fuzzplayground/server.go @@ -18,7 +18,6 @@ import ( func GetPlaygroundServer() *echo.Echo { e := echo.New() - e.Use(middleware.Recover()) e.Use(middleware.Logger()) e.GET("/", indexHandler) diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 6e6f94d9eb..5e188aa57b 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -87,8 +87,12 @@ type TemplateInfo struct { } // NewMockExecuterOptions creates a new mock executeroptions struct -func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions { +func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) (*protocols.ExecutorOptions, error) { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) + dialers, err := protocols.NewDealers(options) + if err != nil { + return nil, err + } executerOpts := &protocols.ExecutorOptions{ TemplateID: info.ID, TemplateInfo: info.Info, @@ -101,9 +105,10 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco Browser: nil, Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Dialers: dialers, } executerOpts.CreateTemplateCtxStore() - return executerOpts + return executerOpts, nil } // NoopWriter is a NooP gologger writer. diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 6a1813efd9..0cc32f5bd3 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -201,11 +201,6 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { } }() - defer func() { - if r := recover(); r != nil { - f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r)) - } - }() if ctx.OnResult == nil { return fmt.Errorf("output callback cannot be nil") From 3dd95f0d6c94566db387bfe7e473b92ffca5305d Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jun 2024 14:01:08 +0200 Subject: [PATCH 4/9] lint --- pkg/protocols/offlinehttp/operators_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/protocols/offlinehttp/operators_test.go b/pkg/protocols/offlinehttp/operators_test.go index 546d988b2c..87650b39b1 100644 --- a/pkg/protocols/offlinehttp/operators_test.go +++ b/pkg/protocols/offlinehttp/operators_test.go @@ -55,6 +55,8 @@ func TestHTTPOperatorMatch(t *testing.T) { Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, } executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + executerOpts.Operators = []*operators.Operators{{}} err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") @@ -125,6 +127,8 @@ func TestHTTPOperatorExtract(t *testing.T) { Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, } executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + executerOpts.Operators = []*operators.Operators{{}} err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") From 2be81c55112255e2f48ceee5eb25e0a35ded3657 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jun 2024 14:58:40 +0200 Subject: [PATCH 5/9] init --- lib/multi.go | 1 + lib/sdk.go | 4 ++++ lib/sdk_private.go | 5 +++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/multi.go b/lib/multi.go index 44c13ddfe6..fffcc96366 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -40,6 +40,7 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: base.parser, + Dialers: base.dialers, } if opts.RateLimitMinute > 0 { opts.RateLimit = opts.RateLimitMinute diff --git a/lib/sdk.go b/lib/sdk.go index 4661f0ccbb..f49c83ec53 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -76,6 +76,7 @@ type NucleiEngine struct { httpClient *retryablehttp.Client parser *templates.Parser authprovider authprovider.AuthProvider + dialers *protocols.Dialers // unexported meta options opts *types.Options @@ -214,6 +215,9 @@ func (e *NucleiEngine) closeInternal() { if e.httpxClient != nil { _ = e.httpxClient.Close() } + if e.dialers != nil { + e.executerOpts.Dialers.Close() + } if e.executerOpts.Dialers != nil { e.executerOpts.Dialers.Close() } diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 0f0115f8e9..786d0e3c3e 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -108,7 +108,8 @@ func (e *NucleiEngine) init(ctx context.Context) error { return err } - dialers, err := protocols.NewDealers(e.opts) + var err error + e.dialers, err = protocols.NewDealers(e.opts) if err != nil { return errors.Wrap(err, "could not create dialers") } @@ -175,7 +176,7 @@ func (e *NucleiEngine) init(ctx context.Context) error { ResumeCfg: types.NewResumeCfg(), Browser: e.browserInstance, Parser: e.parser, - Dialers: dialers, + Dialers: e.dialers, } if len(e.opts.SecretsFile) > 0 { authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts) From ae26e495bb1b7a0181f2766510ea3e7bda00bf1b Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Thu, 20 Jun 2024 17:03:17 +0200 Subject: [PATCH 6/9] dns client --- pkg/operators/common/dsl/dsl.go | 6 +- pkg/protocols/common/protocolinit/init.go | 4 -- pkg/protocols/dns/dns.go | 29 +++++----- pkg/protocols/dns/dnsclientpool/clientpool.go | 56 ++----------------- pkg/protocols/protocols.go | 17 ++++++ 5 files changed, 37 insertions(+), 75 deletions(-) diff --git a/pkg/operators/common/dsl/dsl.go b/pkg/operators/common/dsl/dsl.go index 1b3a02bed9..d31f038dfa 100644 --- a/pkg/operators/common/dsl/dsl.go +++ b/pkg/operators/common/dsl/dsl.go @@ -61,11 +61,7 @@ func init() { return nil, fmt.Errorf("invalid dns type") } - err := dnsclientpool.Init(&types.Options{}) - if err != nil { - return nil, err - } - dnsClient, err := dnsclientpool.Get(nil, &dnsclientpool.Configuration{}) + dnsClient, err := dnsclientpool.Get(&types.Options{}, &dnsclientpool.Configuration{Retries: 11}) if err != nil { return nil, err } diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go index c02d46309a..882feadc5d 100644 --- a/pkg/protocols/common/protocolinit/init.go +++ b/pkg/protocols/common/protocolinit/init.go @@ -3,7 +3,6 @@ package protocolinit import ( "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -14,9 +13,6 @@ func Init(options *types.Options) error { if err := protocolstate.Init(options); err != nil { return err } - if err := dnsclientpool.Init(options); err != nil { - return err - } if err := httpclientpool.Init(options); err != nil { return err } diff --git a/pkg/protocols/dns/dns.go b/pkg/protocols/dns/dns.go index 0a1bbca6cb..bda7e7a4af 100644 --- a/pkg/protocols/dns/dns.go +++ b/pkg/protocols/dns/dns.go @@ -189,25 +189,26 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) { - dnsClientOptions := &dnsclientpool.Configuration{ - Retries: request.Retries, - } - if len(request.Resolvers) > 0 { - if len(request.Resolvers) > 0 { - for _, resolver := range request.Resolvers { - if expressions.ContainsUnresolvedVariables(resolver) != nil { - var err error - resolver, err = expressions.Evaluate(resolver, metadata) - if err != nil { - return nil, errors.Wrap(err, "could not resolve resolvers expressions") - } - dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver) + // if retries are set or resolvers are set, we need to create a new client + if request.Retries > 0 || len(request.Resolvers) > 0 { + dnsClientOptions := &dnsclientpool.Configuration{ + Retries: request.Retries, + } + for _, resolver := range request.Resolvers { + if expressions.ContainsUnresolvedVariables(resolver) != nil { + var err error + resolver, err = expressions.Evaluate(resolver, metadata) + if err != nil { + return nil, errors.Wrap(err, "could not resolve resolvers expressions") } + dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver) } } dnsClientOptions.Resolvers = request.Resolvers + return dnsclientpool.Get(options.Options, dnsClientOptions) } - return dnsclientpool.Get(options.Options, dnsClientOptions) + // otherwise we return the default dns client + return options.Dialers.Dns(), nil } // Requests returns the total number of requests the YAML rule will perform diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go index 4f019808fe..a2c1148dd6 100644 --- a/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -1,21 +1,11 @@ package dnsclientpool import ( - "strconv" - "strings" - "sync" - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryabledns" ) -var ( - poolMutex *sync.RWMutex - normalClient *retryabledns.Client - clientPool map[string]*retryabledns.Client -) - // defaultResolvers contains the list of resolvers known to be trusted. var defaultResolvers = []string{ "1.1.1.1:53", // Cloudflare @@ -24,25 +14,12 @@ var defaultResolvers = []string{ "8.8.4.4:53", // Google } -// Init initializes the client pool implementation -func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } - poolMutex = &sync.RWMutex{} - clientPool = make(map[string]*retryabledns.Client) - +func GetResolversOrDefault(options *types.Options) []string { resolvers := defaultResolvers if len(options.InternalResolversList) > 0 { resolvers = options.InternalResolversList } - var err error - normalClient, err = retryabledns.New(resolvers, 1) - if err != nil { - return errors.Wrap(err, "could not create dns client") - } - return nil + return resolvers } // Configuration contains the custom configuration options for a client @@ -53,30 +30,9 @@ type Configuration struct { Resolvers []string } -// Hash returns the hash of the configuration to allow client pooling -func (c *Configuration) Hash() string { - builder := &strings.Builder{} - builder.WriteString("r") - builder.WriteString(strconv.Itoa(c.Retries)) - builder.WriteString("l") - builder.WriteString(strings.Join(c.Resolvers, "")) - hash := builder.String() - return hash -} - -// Get creates or gets a client for the protocol based on custom configuration +// Get creates a new dns client with scoped usage +// all other usages should rely on ExecuterOptions.Dialers.dnsClient func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { - if !(configuration.Retries > 1) && len(configuration.Resolvers) == 0 { - return normalClient, nil - } - hash := configuration.Hash() - poolMutex.RLock() - if client, ok := clientPool[hash]; ok { - poolMutex.RUnlock() - return client, nil - } - poolMutex.RUnlock() - resolvers := defaultResolvers if len(options.InternalResolversList) > 0 { resolvers = options.InternalResolversList @@ -87,9 +43,5 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl if err != nil { return nil, errors.Wrap(err, "could not create dns client") } - - poolMutex.Lock() - clientPool[hash] = client - poolMutex.Unlock() return client, nil } diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index fbf300f450..3bf5b1ab0e 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/rdap" + "github.com/projectdiscovery/retryabledns" mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" @@ -33,6 +34,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/scan" @@ -135,17 +137,21 @@ type ExecutorOptions struct { type Dialers struct { fastDialer *fastdialer.Dialer rdapClient *rdap.Client + dnsClient *retryabledns.Client } func NewDealers(options *types.Options) (*Dialers, error) { var dialers = &Dialers{} var err error + // allocate once various standard clients + // - fastdialer dialers.fastDialer, err = protocolstate.GetDialerFromOptions(options) if err != nil { return nil, err } + // - rdap dialers.rdapClient = &rdap.Client{} if options.HasDebug() { dialers.rdapClient.Verbose = func(text string) { @@ -153,6 +159,13 @@ func NewDealers(options *types.Options) (*Dialers, error) { } } + // - dns + resolvers := dnsclientpool.GetResolversOrDefault(options) + dialers.dnsClient, err = retryabledns.New(resolvers, 1) + if err != nil { + return nil, err + } + return dialers, nil } @@ -164,6 +177,10 @@ func (d *Dialers) Rdap() *rdap.Client { return d.rdapClient } +func (d *Dialers) Dns() *retryabledns.Client { + return d.dnsClient +} + func (d *Dialers) Close() { if d.fastDialer != nil { d.fastDialer.Close() From fd229abb299f3b7cbfc8efa5f873b48262a8161a Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Thu, 20 Jun 2024 19:41:16 +0200 Subject: [PATCH 7/9] wip: moving http client --- internal/runner/runner.go | 1 + lib/sdk_private.go | 1 + .../common/automaticscan/automaticscan.go | 17 ++-- pkg/protocols/http/http.go | 17 ++-- .../http/httpclientpool/clientpool.go | 87 +++---------------- pkg/protocols/http/request.go | 2 +- pkg/protocols/protocols.go | 26 +++++- 7 files changed, 62 insertions(+), 89 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 6f82acfc5c..36b866d67a 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -183,6 +183,7 @@ func New(options *types.Options) (*Runner, error) { var httpclient *retryablehttp.Client if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { var err error + // we use an independent instance of retryablehttp for internal operations httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}) if err != nil { return nil, err diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 786d0e3c3e..bf292313af 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -115,6 +115,7 @@ func (e *NucleiEngine) init(ctx context.Context) error { } if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { + // we use an independent instance of retryablehttp for internal operations httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) if err != nil { return err diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index a5e51c177a..e19a9a5c4d 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -95,14 +95,21 @@ func New(opts Options) (*Service, error) { return nil, err } - httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{ + httpConnectionOptions := &httpclientpool.Configuration{ Connection: &httpclientpool.ConnectionConfiguration{ DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options), }, - }) - if err != nil { - return nil, errors.Wrap(err, "could not get http client") } + var httpClient *retryablehttp.Client + if httpConnectionOptions.HasStandardOptions() { + httpClient = opts.ExecuterOpts.Dialers.Http() + } else { + httpClient, err = httpclientpool.Get(opts.ExecuterOpts.Options, httpConnectionOptions) + if err != nil { + return nil, errors.Wrap(err, "could not get http client") + } + } + return &Service{ opts: opts.ExecuterOpts, store: opts.Store, @@ -110,7 +117,7 @@ func New(opts Options) (*Service, error) { target: opts.Target, wappalyzer: wappalyzer, templateDirs: templateDirs, // fix this - httpclient: httpclient, + httpclient: httpClient, technologyMappings: mappingData, techTemplates: techDetectTemplates, ServiceOpts: opts, diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index c20cbbfa66..dbf7e3a162 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -312,12 +312,18 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.connConfiguration = connectionConfiguration - client, err := httpclientpool.Get(options.Options, connectionConfiguration) - if err != nil { - return errors.Wrap(err, "could not get dns client") + var httpClient *retryablehttp.Client + if connectionConfiguration.HasStandardOptions() { + httpClient = options.Dialers.Http() + } else { + var err error + httpClient, err = httpclientpool.Get(options.Options, connectionConfiguration) + if err != nil { + return errors.Wrap(err, "could not get dns client") + } } request.customHeaders = make(map[string]string) - request.httpClient = client + request.httpClient = httpClient request.options = options for _, option := range request.options.Options.CustomHeaders { parts := strings.SplitN(option, ":", 2) @@ -336,7 +342,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = options.Dialers.RawHttp() } if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators @@ -411,6 +417,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { + var err error request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index b4ca5d40cf..95bcdd4404 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -7,8 +7,6 @@ import ( "net/http" "net/http/cookiejar" "net/url" - "strconv" - "strings" "sync" "time" @@ -23,14 +21,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" - mapsutil "github.com/projectdiscovery/utils/maps" ) var ( - rawHttpClient *rawhttp.Client forceMaxRedirects int - normalClient *retryablehttp.Client - clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] // MaxResponseHeaderTimeout is the timeout for response headers // to be read from the server (this prevents infinite hang started by server if any) // Note: this will be overridden temporarily when using @timeout request annotation @@ -46,25 +40,12 @@ func GetHttpTimeout(opts *types.Options) time.Duration { // Init initializes the clientpool implementation func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } if options.Timeout > 10 { MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second } if options.ShouldFollowHTTPRedirects() { forceMaxRedirects = options.MaxRedirects } - clientPool = &mapsutil.SyncLockMap[string, *retryablehttp.Client]{ - Map: make(mapsutil.Map[string, *retryablehttp.Client]), - } - - client, err := wrappedGet(options, &Configuration{}) - if err != nil { - return err - } - normalClient = client return nil } @@ -115,67 +96,29 @@ type Configuration struct { ResponseHeaderTimeout time.Duration } -// Hash returns the hash of the configuration to allow client pooling -func (c *Configuration) Hash() string { - builder := &strings.Builder{} - builder.Grow(16) - builder.WriteString("t") - builder.WriteString(strconv.Itoa(c.Threads)) - builder.WriteString("m") - builder.WriteString(strconv.Itoa(c.MaxRedirects)) - builder.WriteString("n") - builder.WriteString(strconv.FormatBool(c.NoTimeout)) - builder.WriteString("f") - builder.WriteString(strconv.Itoa(int(c.RedirectFlow))) - builder.WriteString("r") - builder.WriteString(strconv.FormatBool(c.DisableCookie)) - builder.WriteString("c") - builder.WriteString(strconv.FormatBool(c.Connection != nil)) - builder.WriteString("r") - builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10)) - hash := builder.String() - return hash -} - // HasStandardOptions checks whether the configuration requires custom settings func (c *Configuration) HasStandardOptions() bool { return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0 } -// GetRawHTTP returns the rawhttp request client -func GetRawHTTP(options *types.Options) *rawhttp.Client { - if rawHttpClient == nil { - rawHttpOptions := rawhttp.DefaultOptions - if types.ProxyURL != "" { - rawHttpOptions.Proxy = types.ProxyURL - } else if types.ProxySocksURL != "" { - rawHttpOptions.Proxy = types.ProxySocksURL - } else if protocolstate.Dialer != nil { - rawHttpOptions.FastDialer = protocolstate.Dialer - } - rawHttpOptions.Timeout = GetHttpTimeout(options) - rawHttpClient = rawhttp.NewClient(rawHttpOptions) - } - return rawHttpClient -} - -// Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { - if configuration.HasStandardOptions() { - return normalClient, nil +// GetRawH returns the rawhttp request client +func GetRaw(options *types.Options) *rawhttp.Client { + rawHttpOptions := rawhttp.DefaultOptions + if types.ProxyURL != "" { + rawHttpOptions.Proxy = types.ProxyURL + } else if types.ProxySocksURL != "" { + rawHttpOptions.Proxy = types.ProxySocksURL + } else if protocolstate.Dialer != nil { + rawHttpOptions.FastDialer = protocolstate.Dialer } - return wrappedGet(options, configuration) + rawHttpOptions.Timeout = GetHttpTimeout(options) + return rawhttp.NewClient(rawHttpOptions) } // wrappedGet wraps a get operation without normal client check -func wrappedGet(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { +func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { var err error - hash := configuration.Hash() - if client, ok := clientPool.Get(hash); ok { - return client, nil - } - // Multiple Host retryableHttpOptions := retryablehttp.DefaultOptionsSpraying disableKeepAlives := true @@ -316,12 +259,6 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } client.CheckRetry = retryablehttp.HostSprayRetryPolicy() - // Only add to client pool if we don't have a cookie jar in place. - if jar == nil { - if err := clientPool.Set(hash, client); err != nil { - return nil, err - } - } return client, nil } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 7e5e069348..27593789cb 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -783,7 +783,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout } - if modifiedConfig != nil { + if modifiedConfig != nil && !modifiedConfig.HasStandardOptions() { client, err := httpclientpool.Get(request.options.Options, modifiedConfig) if err != nil { return errors.Wrap(err, "could not get http client") diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 3bf5b1ab0e..336a3f46c1 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -8,8 +8,10 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/rdap" "github.com/projectdiscovery/retryabledns" + "github.com/projectdiscovery/retryablehttp-go" mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" @@ -36,6 +38,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -135,9 +138,11 @@ type ExecutorOptions struct { } type Dialers struct { - fastDialer *fastdialer.Dialer - rdapClient *rdap.Client - dnsClient *retryabledns.Client + fastDialer *fastdialer.Dialer + rdapClient *rdap.Client + dnsClient *retryabledns.Client + httpClient *retryablehttp.Client + rawHttpClient *rawhttp.Client } func NewDealers(options *types.Options) (*Dialers, error) { @@ -166,6 +171,13 @@ func NewDealers(options *types.Options) (*Dialers, error) { return nil, err } + // - http + dialers.httpClient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}) + if err != nil { + return nil, err + } + dialers.rawHttpClient = httpclientpool.GetRaw(options) + return dialers, nil } @@ -181,6 +193,14 @@ func (d *Dialers) Dns() *retryabledns.Client { return d.dnsClient } +func (d *Dialers) Http() *retryablehttp.Client { + return d.httpClient +} + +func (d *Dialers) RawHttp() *rawhttp.Client { + return d.rawHttpClient +} + func (d *Dialers) Close() { if d.fastDialer != nil { d.fastDialer.Close() From f8e190982e28e5b63b1a02829cfc3da77b823319 Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Fri, 21 Jun 2024 00:41:03 +0200 Subject: [PATCH 8/9] started removing protocolstate.Dialer --- pkg/protocols/network/request.go | 3 +-- pkg/protocols/protocols.go | 2 +- pkg/reporting/exporters/es/elasticsearch.go | 25 +++++---------------- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 62807abf93..8665e772e7 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -24,7 +24,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" @@ -63,7 +62,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err errs = append(errs, err) continue } - conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr) + conn, err := request.dialer.Dial(target.Context(), "tcp", addr) if err != nil { errs = append(errs, err) continue diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 336a3f46c1..a1d7f32a62 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -171,7 +171,7 @@ func NewDealers(options *types.Options) (*Dialers, error) { return nil, err } - // - http + // - standard http + raw http dialers.httpClient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}) if err != nil { return nil, err diff --git a/pkg/reporting/exporters/es/elasticsearch.go b/pkg/reporting/exporters/es/elasticsearch.go index 7b627492c9..8225d8a216 100644 --- a/pkg/reporting/exporters/es/elasticsearch.go +++ b/pkg/reporting/exporters/es/elasticsearch.go @@ -2,7 +2,6 @@ package es import ( "bytes" - "crypto/tls" "encoding/base64" "encoding/json" "fmt" @@ -13,7 +12,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/useragent" ) @@ -56,20 +54,9 @@ type Exporter struct { func New(option *Options) (*Exporter, error) { var ei *Exporter - var client *http.Client - if option.HttpClient != nil { - client = option.HttpClient.HTTPClient - } else { - client = &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 10, - MaxIdleConnsPerHost: 10, - DialContext: protocolstate.Dialer.Dial, - DialTLSContext: protocolstate.Dialer.DialTLS, - TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, - }, - } + // we do not allow nil http client + if option.HttpClient == nil { + return nil, errors.New("http client is nil") } // preparing url for elasticsearch @@ -98,7 +85,7 @@ func New(option *Options) (*Exporter, error) { ei = &Exporter{ url: url, authentication: authentication, - elasticsearch: client, + elasticsearch: option.HttpClient.HTTPClient, } return ei, nil } @@ -131,8 +118,8 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { if err != nil { return err } - defer res.Body.Close() - + defer res.Body.Close() + b, err = io.ReadAll(res.Body) if err != nil { return errors.New(err.Error() + "error thrown by elasticsearch " + string(b)) From 6aaabb4544bd35d4399ec98c63ff8284cb39ccb7 Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Fri, 21 Jun 2024 02:50:01 +0200 Subject: [PATCH 9/9] . --- cmd/integration-test/library.go | 2 +- internal/runner/runner.go | 2 +- lib/sdk_private.go | 2 +- pkg/protocols/http/http.go | 3 ++- pkg/protocols/http/request.go | 4 ++-- pkg/protocols/protocols.go | 2 +- pkg/reporting/exporters/splunk/splunkhec.go | 22 ++++----------------- pkg/testutils/testutils.go | 2 +- pkg/tmplexec/flow/flow_executor_test.go | 6 ++++++ pkg/tmplexec/multiproto/multi_test.go | 2 +- 10 files changed, 20 insertions(+), 27 deletions(-) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index 4aba74a040..d5185d936d 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -100,7 +100,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) ratelimiter := ratelimit.New(context.Background(), 150, time.Second) defer ratelimiter.Stop() - dialers, err := protocols.NewDealers(defaultOpts) + dialers, err := protocols.NewDialers(defaultOpts) if err != nil { return nil, errors.Wrap(err, "could not create dialers") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 36b866d67a..daba5fd149 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -454,7 +454,7 @@ func (r *Runner) RunEnumeration() error { fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency) r.fuzzFrequencyCache = fuzzFreqCache - dialers, err := protocols.NewDealers(r.options) + dialers, err := protocols.NewDialers(r.options) if err != nil { return errors.Wrap(err, "could not create dialers") } diff --git a/lib/sdk_private.go b/lib/sdk_private.go index bf292313af..c2551b1276 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -109,7 +109,7 @@ func (e *NucleiEngine) init(ctx context.Context) error { } var err error - e.dialers, err = protocols.NewDealers(e.opts) + e.dialers, err = protocols.NewDialers(e.opts) if err != nil { return errors.Wrap(err, "could not create dialers") } diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index dbf7e3a162..329c59c541 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -319,9 +319,10 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { var err error httpClient, err = httpclientpool.Get(options.Options, connectionConfiguration) if err != nil { - return errors.Wrap(err, "could not get dns client") + return errors.Wrap(err, "could not get http client") } } + request.customHeaders = make(map[string]string) request.httpClient = httpClient request.options = options diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 27593789cb..f1662124fb 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -843,7 +843,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if input.MetaInput.CustomIP != "" { outputEvent["ip"] = input.MetaInput.CustomIP } else { - outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + outputEvent["ip"] = request.options.Dialers.Default().GetDialedIP(hostname) } if len(generatedRequest.interactshURLs) > 0 { @@ -939,7 +939,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if input.MetaInput.CustomIP != "" { outputEvent["ip"] = input.MetaInput.CustomIP } else { - outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + outputEvent["ip"] = request.options.Dialers.Default().GetDialedIP(hostname) } if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent) diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index a1d7f32a62..a42a0db8ac 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -145,7 +145,7 @@ type Dialers struct { rawHttpClient *rawhttp.Client } -func NewDealers(options *types.Options) (*Dialers, error) { +func NewDialers(options *types.Options) (*Dialers, error) { var dialers = &Dialers{} var err error diff --git a/pkg/reporting/exporters/splunk/splunkhec.go b/pkg/reporting/exporters/splunk/splunkhec.go index e448d6d2fc..d24c960ebb 100644 --- a/pkg/reporting/exporters/splunk/splunkhec.go +++ b/pkg/reporting/exporters/splunk/splunkhec.go @@ -2,17 +2,14 @@ package splunk import ( "bytes" - "crypto/tls" "encoding/json" "fmt" "io" "net" "net/http" - "time" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/useragent" ) @@ -48,20 +45,9 @@ type Exporter struct { func New(option *Options) (*Exporter, error) { var ei *Exporter - var client *http.Client - if option.HttpClient != nil { - client = option.HttpClient.HTTPClient - } else { - client = &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 10, - MaxIdleConnsPerHost: 10, - DialContext: protocolstate.Dialer.Dial, - DialTLSContext: protocolstate.Dialer.DialTLS, - TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, - }, - } + // we do not allow nil http client + if option.HttpClient == nil { + return nil, errors.New("http client is nil") } // preparing url for splunk @@ -85,7 +71,7 @@ func New(option *Options) (*Exporter, error) { ei = &Exporter{ url: url, authentication: authentication, - splunk: client, + splunk: option.HttpClient.HTTPClient, } return ei, nil } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 5e188aa57b..20c7fdbb8c 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -89,7 +89,7 @@ type TemplateInfo struct { // NewMockExecuterOptions creates a new mock executeroptions struct func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) (*protocols.ExecutorOptions, error) { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) - dialers, err := protocols.NewDealers(options) + dialers, err := protocols.NewDialers(options) if err != nil { return nil, err } diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index b47b38a2a0..cef962a188 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -26,6 +26,11 @@ func setup() { testutils.Init(options) progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) + dialers, err := protocols.NewDialers(options) + if err != nil { + log.Fatalf("Could not create dialers: %s\n", err) + } + executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, @@ -36,6 +41,7 @@ func setup() { Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), Parser: templates.NewParser(), + Dialers: dialers, } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index cf90cc850b..bb47e0adad 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -25,7 +25,7 @@ func setup(t *testing.T) { testutils.Init(options) progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) - dialers, err := protocols.NewDealers(options) + dialers, err := protocols.NewDialers(options) require.Nil(t, err, "could not create dialers") executerOpts = protocols.ExecutorOptions{