diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index 1324688e05..d5185d936d 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -100,6 +100,12 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) ratelimiter := ratelimit.New(context.Background(), 150, time.Second) defer ratelimiter.Stop() + dialers, err := protocols.NewDialers(defaultOpts) + if err != nil { + return nil, errors.Wrap(err, "could not create dialers") + } + defer dialers.Close() + executerOpts := protocols.ExecutorOptions{ Output: outputWriter, Options: defaultOpts, @@ -112,6 +118,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: templates.NewParser(), + Dialers: dialers, } engine := core.New(defaultOpts) engine.SetExecuterOptions(executerOpts) diff --git a/internal/runner/options.go b/internal/runner/options.go index 8798c73113..ebb49fff7e 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -32,7 +32,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" fileutil "github.com/projectdiscovery/utils/file" "github.com/projectdiscovery/utils/generic" - logutil "github.com/projectdiscovery/utils/log" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -342,7 +341,7 @@ func configureOutput(options *types.Options) { } // disable standard logger (ref: https://github.com/golang/go/issues/19895) - logutil.DisableDefaultLogger() + // logutil.DisableDefaultLogger() } // loadResolvers loads resolvers from both user-provided flags and file diff --git a/internal/runner/runner.go b/internal/runner/runner.go index bc436500af..daba5fd149 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -183,6 +183,7 @@ func New(options *types.Options) (*Runner, error) { var httpclient *retryablehttp.Client if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { var err error + // we use an independent instance of retryablehttp for internal operations httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}) if err != nil { return nil, err @@ -453,6 +454,11 @@ func (r *Runner) RunEnumeration() error { fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency) r.fuzzFrequencyCache = fuzzFreqCache + dialers, err := protocols.NewDialers(r.options) + if err != nil { + return errors.Wrap(err, "could not create dialers") + } + // Create the executor options which will be used throughout the execution // stage by the nuclei engine modules. executorOpts := protocols.ExecutorOptions{ @@ -472,6 +478,7 @@ func (r *Runner) RunEnumeration() error { TemporaryDirectory: r.tmpDir, Parser: r.parser, FuzzParamsFrequency: fuzzFreqCache, + Dialers: dialers, } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { diff --git a/lib/multi.go b/lib/multi.go index 44c13ddfe6..fffcc96366 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -40,6 +40,7 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: base.parser, + Dialers: base.dialers, } if opts.RateLimitMinute > 0 { opts.RateLimit = opts.RateLimitMinute diff --git a/lib/sdk.go b/lib/sdk.go index 60f5255bcd..f49c83ec53 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -76,6 +76,7 @@ type NucleiEngine struct { httpClient *retryablehttp.Client parser *templates.Parser authprovider authprovider.AuthProvider + dialers *protocols.Dialers // unexported meta options opts *types.Options @@ -214,6 +215,12 @@ func (e *NucleiEngine) closeInternal() { if e.httpxClient != nil { _ = e.httpxClient.Close() } + if e.dialers != nil { + e.executerOpts.Dialers.Close() + } + if e.executerOpts.Dialers != nil { + e.executerOpts.Dialers.Close() + } } // Close all resources used by nuclei engine diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 3475200bd0..c2551b1276 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -108,7 +108,14 @@ func (e *NucleiEngine) init(ctx context.Context) error { return err } + var err error + e.dialers, err = protocols.NewDialers(e.opts) + if err != nil { + return errors.Wrap(err, "could not create dialers") + } + if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { + // we use an independent instance of retryablehttp for internal operations httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) if err != nil { return err @@ -127,7 +134,6 @@ func (e *NucleiEngine) init(ctx context.Context) error { }) e.applyRequiredDefaults(ctx) - var err error // setup progressbar if e.enableStats { @@ -171,6 +177,7 @@ func (e *NucleiEngine) init(ctx context.Context) error { ResumeCfg: types.NewResumeCfg(), Browser: e.browserInstance, Parser: e.parser, + Dialers: e.dialers, } if len(e.opts.SecretsFile) > 0 { authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts) diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 367bd013b2..825dd7eea3 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -117,11 +117,6 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() return ExecuteProgram(program, args, opts) }) if err != nil { diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go index 17d9f746f5..53943efcb7 100644 --- a/pkg/js/compiler/pool.go +++ b/pkg/js/compiler/pool.go @@ -84,11 +84,6 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg opts.Cleanup(runtime) } }() - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %s", r) - } - }() // set template ctx _ = runtime.Set("template", args.TemplateCtx) // set args diff --git a/pkg/operators/common/dsl/dsl.go b/pkg/operators/common/dsl/dsl.go index 1b3a02bed9..d31f038dfa 100644 --- a/pkg/operators/common/dsl/dsl.go +++ b/pkg/operators/common/dsl/dsl.go @@ -61,11 +61,7 @@ func init() { return nil, fmt.Errorf("invalid dns type") } - err := dnsclientpool.Init(&types.Options{}) - if err != nil { - return nil, err - } - dnsClient, err := dnsclientpool.Get(nil, &dnsclientpool.Configuration{}) + dnsClient, err := dnsclientpool.Get(&types.Options{}, &dnsclientpool.Configuration{Retries: 11}) if err != nil { return nil, err } diff --git a/pkg/protocols/code/code_test.go b/pkg/protocols/code/code_test.go index 320f7c5481..29bc0af00c 100644 --- a/pkg/protocols/code/code_test.go +++ b/pkg/protocols/code/code_test.go @@ -24,11 +24,14 @@ func TestCodeProtocol(t *testing.T) { Engine: []string{"sh"}, Source: "echo test", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile code request") var gotEvent output.InternalEvent diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index a5e51c177a..e19a9a5c4d 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -95,14 +95,21 @@ func New(opts Options) (*Service, error) { return nil, err } - httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{ + httpConnectionOptions := &httpclientpool.Configuration{ Connection: &httpclientpool.ConnectionConfiguration{ DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options), }, - }) - if err != nil { - return nil, errors.Wrap(err, "could not get http client") } + var httpClient *retryablehttp.Client + if httpConnectionOptions.HasStandardOptions() { + httpClient = opts.ExecuterOpts.Dialers.Http() + } else { + httpClient, err = httpclientpool.Get(opts.ExecuterOpts.Options, httpConnectionOptions) + if err != nil { + return nil, errors.Wrap(err, "could not get http client") + } + } + return &Service{ opts: opts.ExecuterOpts, store: opts.Store, @@ -110,7 +117,7 @@ func New(opts Options) (*Service, error) { target: opts.Target, wappalyzer: wappalyzer, templateDirs: templateDirs, // fix this - httpclient: httpclient, + httpclient: httpClient, technologyMappings: mappingData, techTemplates: techDetectTemplates, ServiceOpts: opts, diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go index c8268337f5..882feadc5d 100644 --- a/pkg/protocols/common/protocolinit/init.go +++ b/pkg/protocols/common/protocolinit/init.go @@ -3,11 +3,8 @@ package protocolinit import ( "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/types" ) @@ -16,21 +13,12 @@ func Init(options *types.Options) error { if err := protocolstate.Init(options); err != nil { return err } - if err := dnsclientpool.Init(options); err != nil { - return err - } if err := httpclientpool.Init(options); err != nil { return err } if err := signerpool.Init(options); err != nil { return err } - if err := networkclientpool.Init(options); err != nil { - return err - } - if err := rdapclientpool.Init(options); err != nil { - return err - } if err := compiler.Init(options); err != nil { return err } diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index cf6f5234ec..07eb93e75a 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -26,12 +26,7 @@ func ShouldInit() bool { return Dialer == nil } -// Init creates the Dialer instance based on user configuration -func Init(options *types.Options) error { - if Dialer != nil { - return nil - } - +func GetDialerFromOptions(options *types.Options) (*fastdialer.Dialer, error) { lfaAllowed = options.AllowLocalFileAccess opts := fastdialer.DefaultOptions if options.DialerTimeout > 0 { @@ -66,7 +61,7 @@ func Init(options *types.Options) error { case options.SourceIP != "" && options.Interface != "": isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, options.Interface) if err != nil { - return err + return nil, err } if isAssociated { opts.Dialer = &net.Dialer{ @@ -75,12 +70,12 @@ func Init(options *types.Options) error { }, } } else { - return fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface) + return nil, fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface) } case options.SourceIP != "": isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, "any") if err != nil { - return err + return nil, err } if isAssociated { opts.Dialer = &net.Dialer{ @@ -89,12 +84,12 @@ func Init(options *types.Options) error { }, } } else { - return fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP) + return nil, fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP) } case options.Interface != "": ifadrr, err := interfaceAddress(options.Interface) if err != nil { - return err + return nil, err } opts.Dialer = &net.Dialer{ LocalAddr: &net.TCPAddr{ @@ -105,7 +100,7 @@ func Init(options *types.Options) error { if types.ProxySocksURL != "" { proxyURL, err := url.Parse(types.ProxySocksURL) if err != nil { - return err + return nil, err } var forward *net.Dialer if opts.Dialer != nil { @@ -119,7 +114,7 @@ func Init(options *types.Options) error { } dialer, err := proxy.FromURL(proxyURL, forward) if err != nil { - return err + return nil, err } opts.ProxyDialer = &dialer } @@ -143,9 +138,26 @@ func Init(options *types.Options) error { // fastdialer now by default fallbacks to ztls when there are tls related errors dialer, err := fastdialer.NewDialer(opts) if err != nil { - return errors.Wrap(err, "could not create dialer") + return nil, errors.Wrap(err, "could not create dialer") + } + + return dialer, nil +} + +// Init creates the Dialer instance based on user configuration +func Init(options *types.Options) error { + // *************************** + // todo: remove this part + if Dialer != nil { + return nil + } + + dialer, err := GetDialerFromOptions(options) + if err != nil { + return err } Dialer = dialer + // ********************+***** // override dialer in mysql mysql.RegisterDialContext("tcp", func(ctx context.Context, addr string) (net.Conn, error) { diff --git a/pkg/protocols/dns/dns.go b/pkg/protocols/dns/dns.go index 0a1bbca6cb..bda7e7a4af 100644 --- a/pkg/protocols/dns/dns.go +++ b/pkg/protocols/dns/dns.go @@ -189,25 +189,26 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) { - dnsClientOptions := &dnsclientpool.Configuration{ - Retries: request.Retries, - } - if len(request.Resolvers) > 0 { - if len(request.Resolvers) > 0 { - for _, resolver := range request.Resolvers { - if expressions.ContainsUnresolvedVariables(resolver) != nil { - var err error - resolver, err = expressions.Evaluate(resolver, metadata) - if err != nil { - return nil, errors.Wrap(err, "could not resolve resolvers expressions") - } - dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver) + // if retries are set or resolvers are set, we need to create a new client + if request.Retries > 0 || len(request.Resolvers) > 0 { + dnsClientOptions := &dnsclientpool.Configuration{ + Retries: request.Retries, + } + for _, resolver := range request.Resolvers { + if expressions.ContainsUnresolvedVariables(resolver) != nil { + var err error + resolver, err = expressions.Evaluate(resolver, metadata) + if err != nil { + return nil, errors.Wrap(err, "could not resolve resolvers expressions") } + dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver) } } dnsClientOptions.Resolvers = request.Resolvers + return dnsclientpool.Get(options.Options, dnsClientOptions) } - return dnsclientpool.Get(options.Options, dnsClientOptions) + // otherwise we return the default dns client + return options.Dialers.Dns(), nil } // Requests returns the total number of requests the YAML rule will perform diff --git a/pkg/protocols/dns/dns_test.go b/pkg/protocols/dns/dns_test.go index bf2c3a9f76..392c13e0c5 100644 --- a/pkg/protocols/dns/dns_test.go +++ b/pkg/protocols/dns/dns_test.go @@ -24,11 +24,13 @@ func TestDNSCompileMake(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req, err := request.Make("one.one.one.one", map[string]interface{}{"FQDN": "one.one.one.one"}) @@ -53,11 +55,14 @@ func TestDNSRequests(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") reqCount := request.Requests() @@ -76,11 +81,15 @@ func TestDNSRequests(t *testing.T) { Name: "{{subdomain}}.{{FQDN}}", Payloads: map[string]interface{}{"subdomain": []string{"a", "b", "c"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") reqCount := request.Requests() diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go index 4f019808fe..a2c1148dd6 100644 --- a/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -1,21 +1,11 @@ package dnsclientpool import ( - "strconv" - "strings" - "sync" - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryabledns" ) -var ( - poolMutex *sync.RWMutex - normalClient *retryabledns.Client - clientPool map[string]*retryabledns.Client -) - // defaultResolvers contains the list of resolvers known to be trusted. var defaultResolvers = []string{ "1.1.1.1:53", // Cloudflare @@ -24,25 +14,12 @@ var defaultResolvers = []string{ "8.8.4.4:53", // Google } -// Init initializes the client pool implementation -func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } - poolMutex = &sync.RWMutex{} - clientPool = make(map[string]*retryabledns.Client) - +func GetResolversOrDefault(options *types.Options) []string { resolvers := defaultResolvers if len(options.InternalResolversList) > 0 { resolvers = options.InternalResolversList } - var err error - normalClient, err = retryabledns.New(resolvers, 1) - if err != nil { - return errors.Wrap(err, "could not create dns client") - } - return nil + return resolvers } // Configuration contains the custom configuration options for a client @@ -53,30 +30,9 @@ type Configuration struct { Resolvers []string } -// Hash returns the hash of the configuration to allow client pooling -func (c *Configuration) Hash() string { - builder := &strings.Builder{} - builder.WriteString("r") - builder.WriteString(strconv.Itoa(c.Retries)) - builder.WriteString("l") - builder.WriteString(strings.Join(c.Resolvers, "")) - hash := builder.String() - return hash -} - -// Get creates or gets a client for the protocol based on custom configuration +// Get creates a new dns client with scoped usage +// all other usages should rely on ExecuterOptions.Dialers.dnsClient func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { - if !(configuration.Retries > 1) && len(configuration.Resolvers) == 0 { - return normalClient, nil - } - hash := configuration.Hash() - poolMutex.RLock() - if client, ok := clientPool[hash]; ok { - poolMutex.RUnlock() - return client, nil - } - poolMutex.RUnlock() - resolvers := defaultResolvers if len(options.InternalResolversList) > 0 { resolvers = options.InternalResolversList @@ -87,9 +43,5 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl if err != nil { return nil, errors.Wrap(err, "could not create dns client") } - - poolMutex.Lock() - clientPool[hash] = client - poolMutex.Unlock() return client, nil } diff --git a/pkg/protocols/dns/operators_test.go b/pkg/protocols/dns/operators_test.go index 80d02908de..59a9e688ec 100644 --- a/pkg/protocols/dns/operators_test.go +++ b/pkg/protocols/dns/operators_test.go @@ -31,11 +31,13 @@ func TestResponseToDSLMap(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) @@ -65,11 +67,13 @@ func TestDNSOperatorMatch(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) @@ -177,11 +181,14 @@ func TestDNSOperatorExtract(t *testing.T) { Recursion: &recursion, Name: "{{FQDN}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) @@ -227,10 +234,12 @@ func TestDNSMakeResult(t *testing.T) { recursion := false testutils.Init(options) templateID := "testing-dns" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") request := &Request{ RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", @@ -253,7 +262,7 @@ func TestDNSMakeResult(t *testing.T) { }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") req := new(dns.Msg) diff --git a/pkg/protocols/dns/request_test.go b/pkg/protocols/dns/request_test.go index 81f0b98ab2..a850a04351 100644 --- a/pkg/protocols/dns/request_test.go +++ b/pkg/protocols/dns/request_test.go @@ -22,10 +22,12 @@ func TestDNSExecuteWithResults(t *testing.T) { recursion := false testutils.Init(options) templateID := "testing-dns" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") request := &Request{ RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", @@ -48,7 +50,7 @@ func TestDNSExecuteWithResults(t *testing.T) { }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile dns request") var finalEvent *output.InternalWrappedEvent diff --git a/pkg/protocols/file/find_test.go b/pkg/protocols/file/find_test.go index 3df5d2383b..ea692c1410 100644 --- a/pkg/protocols/file/find_test.go +++ b/pkg/protocols/file/find_test.go @@ -26,11 +26,13 @@ func TestFindInputPaths(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") tempDir, err := os.MkdirTemp("", "test-*") diff --git a/pkg/protocols/file/operators_test.go b/pkg/protocols/file/operators_test.go index 72a0ca45bd..d874533488 100644 --- a/pkg/protocols/file/operators_test.go +++ b/pkg/protocols/file/operators_test.go @@ -40,11 +40,13 @@ func TestResponseToDSLMap(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := "test-data\r\n" @@ -66,11 +68,13 @@ func TestFileOperatorMatch(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := "test-data\r\n1.1.1.1\r\n" @@ -155,11 +159,13 @@ func TestFileOperatorExtract(t *testing.T) { DenyList: []string{".go"}, Operators: newMockOperator(), } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := "test-data\r\n1.1.1.1\r\n" @@ -255,10 +261,13 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi testutils.Init(options) templateID := "testing-file" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + request := &Request{ ID: templateID, MaxSize: "1Gb", @@ -276,7 +285,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") matchedFileName := "test.txt" diff --git a/pkg/protocols/file/request_test.go b/pkg/protocols/file/request_test.go index 7c69cf5bce..a180a30c09 100644 --- a/pkg/protocols/file/request_test.go +++ b/pkg/protocols/file/request_test.go @@ -24,10 +24,14 @@ func TestFileExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-file" - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + request := &Request{ ID: templateID, MaxSize: "1Gb", @@ -49,7 +53,7 @@ func TestFileExecuteWithResults(t *testing.T) { }, options: executerOpts, } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") tempDir, err := os.MkdirTemp("", "test-*") diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 57cc371093..c7285eccc1 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -53,13 +53,6 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var // typically used for waitEvent waitFuncs := make([]func() error, 0) - // avoid any future panics caused due to go-rod library - defer func() { - if r := recover(); r != nil { - err = errorutil.New("panic on headless action: %v", r) - } - }() - for _, act := range actions { switch act.ActionType.ActionType { case ActionNavigate: diff --git a/pkg/protocols/http/build_request_test.go b/pkg/protocols/http/build_request_test.go index 4405bd10b7..700603bdd6 100644 --- a/pkg/protocols/http/build_request_test.go +++ b/pkg/protocols/http/build_request_test.go @@ -31,11 +31,14 @@ func TestMakeRequestFromModal(t *testing.T) { "Content-Length": "1", }, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -61,11 +64,13 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { Path: []string{"{{BaseURL}}?query=example"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -101,11 +106,13 @@ Connection: close Authorization: Basic {{username + ':' + password}} Accept-Encoding: gzip`}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -142,11 +149,13 @@ Connection: close Authorization: Basic {{base64(username + ':' + password)}} Accept-Encoding: gzip`}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) @@ -175,11 +184,13 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") generator := request.newGenerator(false) diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index c20cbbfa66..329c59c541 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -312,12 +312,19 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.connConfiguration = connectionConfiguration - client, err := httpclientpool.Get(options.Options, connectionConfiguration) - if err != nil { - return errors.Wrap(err, "could not get dns client") + var httpClient *retryablehttp.Client + if connectionConfiguration.HasStandardOptions() { + httpClient = options.Dialers.Http() + } else { + var err error + httpClient, err = httpclientpool.Get(options.Options, connectionConfiguration) + if err != nil { + return errors.Wrap(err, "could not get http client") + } } + request.customHeaders = make(map[string]string) - request.httpClient = client + request.httpClient = httpClient request.options = options for _, option := range request.options.Options.CustomHeaders { parts := strings.SplitN(option, ":", 2) @@ -336,7 +343,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = options.Dialers.RawHttp() } if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators @@ -411,6 +418,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { + var err error request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") diff --git a/pkg/protocols/http/http_test.go b/pkg/protocols/http/http_test.go index 396fd55eea..b97c009126 100644 --- a/pkg/protocols/http/http_test.go +++ b/pkg/protocols/http/http_test.go @@ -31,11 +31,13 @@ Connection: close Authorization: Basic {{username + ':' + password}} Accept-Encoding: gzip`}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http request") require.Equal(t, 6, request.Requests(), "could not get correct number of requests") require.Equal(t, map[string]string{"User-Agent": "test", "Hello": "World"}, request.customHeaders, "could not get correct custom headers") diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index b4ca5d40cf..95bcdd4404 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -7,8 +7,6 @@ import ( "net/http" "net/http/cookiejar" "net/url" - "strconv" - "strings" "sync" "time" @@ -23,14 +21,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" - mapsutil "github.com/projectdiscovery/utils/maps" ) var ( - rawHttpClient *rawhttp.Client forceMaxRedirects int - normalClient *retryablehttp.Client - clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] // MaxResponseHeaderTimeout is the timeout for response headers // to be read from the server (this prevents infinite hang started by server if any) // Note: this will be overridden temporarily when using @timeout request annotation @@ -46,25 +40,12 @@ func GetHttpTimeout(opts *types.Options) time.Duration { // Init initializes the clientpool implementation func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } if options.Timeout > 10 { MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second } if options.ShouldFollowHTTPRedirects() { forceMaxRedirects = options.MaxRedirects } - clientPool = &mapsutil.SyncLockMap[string, *retryablehttp.Client]{ - Map: make(mapsutil.Map[string, *retryablehttp.Client]), - } - - client, err := wrappedGet(options, &Configuration{}) - if err != nil { - return err - } - normalClient = client return nil } @@ -115,67 +96,29 @@ type Configuration struct { ResponseHeaderTimeout time.Duration } -// Hash returns the hash of the configuration to allow client pooling -func (c *Configuration) Hash() string { - builder := &strings.Builder{} - builder.Grow(16) - builder.WriteString("t") - builder.WriteString(strconv.Itoa(c.Threads)) - builder.WriteString("m") - builder.WriteString(strconv.Itoa(c.MaxRedirects)) - builder.WriteString("n") - builder.WriteString(strconv.FormatBool(c.NoTimeout)) - builder.WriteString("f") - builder.WriteString(strconv.Itoa(int(c.RedirectFlow))) - builder.WriteString("r") - builder.WriteString(strconv.FormatBool(c.DisableCookie)) - builder.WriteString("c") - builder.WriteString(strconv.FormatBool(c.Connection != nil)) - builder.WriteString("r") - builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10)) - hash := builder.String() - return hash -} - // HasStandardOptions checks whether the configuration requires custom settings func (c *Configuration) HasStandardOptions() bool { return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0 } -// GetRawHTTP returns the rawhttp request client -func GetRawHTTP(options *types.Options) *rawhttp.Client { - if rawHttpClient == nil { - rawHttpOptions := rawhttp.DefaultOptions - if types.ProxyURL != "" { - rawHttpOptions.Proxy = types.ProxyURL - } else if types.ProxySocksURL != "" { - rawHttpOptions.Proxy = types.ProxySocksURL - } else if protocolstate.Dialer != nil { - rawHttpOptions.FastDialer = protocolstate.Dialer - } - rawHttpOptions.Timeout = GetHttpTimeout(options) - rawHttpClient = rawhttp.NewClient(rawHttpOptions) - } - return rawHttpClient -} - -// Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { - if configuration.HasStandardOptions() { - return normalClient, nil +// GetRawH returns the rawhttp request client +func GetRaw(options *types.Options) *rawhttp.Client { + rawHttpOptions := rawhttp.DefaultOptions + if types.ProxyURL != "" { + rawHttpOptions.Proxy = types.ProxyURL + } else if types.ProxySocksURL != "" { + rawHttpOptions.Proxy = types.ProxySocksURL + } else if protocolstate.Dialer != nil { + rawHttpOptions.FastDialer = protocolstate.Dialer } - return wrappedGet(options, configuration) + rawHttpOptions.Timeout = GetHttpTimeout(options) + return rawhttp.NewClient(rawHttpOptions) } // wrappedGet wraps a get operation without normal client check -func wrappedGet(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { +func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { var err error - hash := configuration.Hash() - if client, ok := clientPool.Get(hash); ok { - return client, nil - } - // Multiple Host retryableHttpOptions := retryablehttp.DefaultOptionsSpraying disableKeepAlives := true @@ -316,12 +259,6 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } client.CheckRetry = retryablehttp.HostSprayRetryPolicy() - // Only add to client pool if we don't have a cookie jar in place. - if jar == nil { - if err := clientPool.Set(hash, client); err != nil { - return nil, err - } - } return client, nil } diff --git a/pkg/protocols/http/operators_test.go b/pkg/protocols/http/operators_test.go index deb5319d9b..832a15c533 100644 --- a/pkg/protocols/http/operators_test.go +++ b/pkg/protocols/http/operators_test.go @@ -27,11 +27,13 @@ func TestResponseToDSLMap(t *testing.T) { Path: []string{"{{BaseURL}}?test=1"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -57,11 +59,13 @@ func TestHTTPOperatorMatch(t *testing.T) { Path: []string{"{{BaseURL}}?test=1"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -145,11 +149,13 @@ func TestHTTPOperatorExtract(t *testing.T) { Path: []string{"{{BaseURL}}?test=1"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -272,11 +278,13 @@ func TestHTTPMakeResult(t *testing.T) { }}, }, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 7e5e069348..f1662124fb 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -783,7 +783,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout } - if modifiedConfig != nil { + if modifiedConfig != nil && !modifiedConfig.HasStandardOptions() { client, err := httpclientpool.Get(request.options.Options, modifiedConfig) if err != nil { return errors.Wrap(err, "could not get http client") @@ -843,7 +843,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if input.MetaInput.CustomIP != "" { outputEvent["ip"] = input.MetaInput.CustomIP } else { - outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + outputEvent["ip"] = request.options.Dialers.Default().GetDialedIP(hostname) } if len(generatedRequest.interactshURLs) > 0 { @@ -939,7 +939,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if input.MetaInput.CustomIP != "" { outputEvent["ip"] = input.MetaInput.CustomIP } else { - outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + outputEvent["ip"] = request.options.Dialers.Default().GetDialedIP(hostname) } if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent) diff --git a/pkg/protocols/http/request_test.go b/pkg/protocols/http/request_test.go index c0bd2bb346..76b3a4806b 100644 --- a/pkg/protocols/http/request_test.go +++ b/pkg/protocols/http/request_test.go @@ -71,12 +71,13 @@ Disallow: /c`)) })) defer ts.Close() - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") var finalEvent *output.InternalWrappedEvent @@ -144,12 +145,13 @@ func TestDisableTE(t *testing.T) { })) defer ts.Close() - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile http raw request") err = request2.Compile(executerOpts) @@ -220,10 +222,12 @@ func TestReqURLPattern(t *testing.T) { })) defer ts.Close() - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") client, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress)) executerOpts.Interactsh = client defer client.Close() @@ -235,7 +239,7 @@ func TestReqURLPattern(t *testing.T) { "{{randstr}}": "2eNU2kbrOcUDzhnUL1RGvSo1it7", } - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") var finalEvent *output.InternalWrappedEvent diff --git a/pkg/protocols/network/network.go b/pkg/protocols/network/network.go index 70d618dcb5..23412af799 100644 --- a/pkg/protocols/network/network.go +++ b/pkg/protocols/network/network.go @@ -11,7 +11,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -232,12 +231,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads) } - // Create a client for the class - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errors.Wrap(err, "could not get network client") - } - request.dialer = client + request.dialer = options.Dialers.Default() if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators diff --git a/pkg/protocols/network/network_test.go b/pkg/protocols/network/network_test.go index 83dbc7fc6d..f3a30db804 100644 --- a/pkg/protocols/network/network_test.go +++ b/pkg/protocols/network/network_test.go @@ -21,11 +21,14 @@ func TestNetworkCompileMake(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") require.Equal(t, 1, len(request.addresses), "could not get correct number of input address") diff --git a/pkg/protocols/network/networkclientpool/clientpool.go b/pkg/protocols/network/networkclientpool/clientpool.go deleted file mode 100644 index a67cee2967..0000000000 --- a/pkg/protocols/network/networkclientpool/clientpool.go +++ /dev/null @@ -1,34 +0,0 @@ -package networkclientpool - -import ( - "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v3/pkg/types" -) - -var ( - normalClient *fastdialer.Dialer -) - -// Init initializes the clientpool implementation -func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } - normalClient = protocolstate.Dialer - return nil -} - -// Configuration contains the custom configuration options for a client -type Configuration struct{} - -// Hash returns the hash of the configuration to allow client pooling -func (c *Configuration) Hash() string { - return "" -} - -// Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration /*TODO review unused parameters*/) (*fastdialer.Dialer, error) { - return normalClient, nil -} diff --git a/pkg/protocols/network/operators_test.go b/pkg/protocols/network/operators_test.go index 88bc94b4d6..112997081f 100644 --- a/pkg/protocols/network/operators_test.go +++ b/pkg/protocols/network/operators_test.go @@ -25,11 +25,14 @@ func TestResponseToDSLMap(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data\r\n"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" @@ -50,11 +53,15 @@ func TestNetworkOperatorMatch(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data\r\n"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" @@ -135,11 +142,14 @@ func TestNetworkOperatorExtract(t *testing.T) { ReadSize: 1024, Inputs: []*Input{{Data: "test-data\r\n"}}, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" @@ -198,11 +208,14 @@ func TestNetworkMakeResult(t *testing.T) { }}, }, } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") req := "test-data\r\n" diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 62807abf93..8665e772e7 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -24,7 +24,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" @@ -63,7 +62,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err errs = append(errs, err) continue } - conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr) + conn, err := request.dialer.Dial(target.Context(), "tcp", addr) if err != nil { errs = append(errs, err) continue diff --git a/pkg/protocols/network/request_test.go b/pkg/protocols/network/request_test.go index 1945888e9b..9e2dc06627 100644 --- a/pkg/protocols/network/request_test.go +++ b/pkg/protocols/network/request_test.go @@ -55,10 +55,13 @@ func TestNetworkExecuteWithResults(t *testing.T) { request.Address[0] = "{{Hostname}}" request.Inputs = append(request.Inputs, &Input{Data: fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host)}) - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") diff --git a/pkg/protocols/offlinehttp/find_test.go b/pkg/protocols/offlinehttp/find_test.go index 83249bc97f..08d1d3e31d 100644 --- a/pkg/protocols/offlinehttp/find_test.go +++ b/pkg/protocols/offlinehttp/find_test.go @@ -20,12 +20,14 @@ func TestFindResponses(t *testing.T) { testutils.Init(options) templateID := "testing-offline" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") tempDir, err := os.MkdirTemp("", "test-*") diff --git a/pkg/protocols/offlinehttp/operators_test.go b/pkg/protocols/offlinehttp/operators_test.go index 21bb869392..87650b39b1 100644 --- a/pkg/protocols/offlinehttp/operators_test.go +++ b/pkg/protocols/offlinehttp/operators_test.go @@ -22,12 +22,14 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -48,12 +50,15 @@ func TestHTTPOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -117,12 +122,15 @@ func TestHTTPOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + executerOpts.Operators = []*operators.Operators{{}} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} @@ -171,10 +179,12 @@ func TestHTTPMakeResult(t *testing.T) { testutils.Init(options) templateID := "testing-http" request := &Request{} - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") executerOpts.Operators = []*operators.Operators{{ Matchers: []*matchers.Matcher{{ Name: "test", @@ -188,7 +198,7 @@ func TestHTTPMakeResult(t *testing.T) { Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }} - err := request.Compile(executerOpts) + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") resp := &http.Response{} diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index af7d2b4766..a42a0db8ac 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -5,7 +5,13 @@ import ( "encoding/base64" "sync/atomic" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/rdap" + "github.com/projectdiscovery/retryabledns" + "github.com/projectdiscovery/retryablehttp-go" mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" @@ -27,9 +33,12 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -124,6 +133,81 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool + // shared dialers + Dialers *Dialers +} + +type Dialers struct { + fastDialer *fastdialer.Dialer + rdapClient *rdap.Client + dnsClient *retryabledns.Client + httpClient *retryablehttp.Client + rawHttpClient *rawhttp.Client +} + +func NewDialers(options *types.Options) (*Dialers, error) { + var dialers = &Dialers{} + var err error + + // allocate once various standard clients + // - fastdialer + dialers.fastDialer, err = protocolstate.GetDialerFromOptions(options) + if err != nil { + return nil, err + } + + // - rdap + dialers.rdapClient = &rdap.Client{} + if options.HasDebug() { + dialers.rdapClient.Verbose = func(text string) { + gologger.Debug().Msgf("rdap: %s", text) + } + } + + // - dns + resolvers := dnsclientpool.GetResolversOrDefault(options) + dialers.dnsClient, err = retryabledns.New(resolvers, 1) + if err != nil { + return nil, err + } + + // - standard http + raw http + dialers.httpClient, err = httpclientpool.Get(options, &httpclientpool.Configuration{}) + if err != nil { + return nil, err + } + dialers.rawHttpClient = httpclientpool.GetRaw(options) + + return dialers, nil +} + +func (d *Dialers) Default() *fastdialer.Dialer { + return d.fastDialer +} + +func (d *Dialers) Rdap() *rdap.Client { + return d.rdapClient +} + +func (d *Dialers) Dns() *retryabledns.Client { + return d.dnsClient +} + +func (d *Dialers) Http() *retryablehttp.Client { + return d.httpClient +} + +func (d *Dialers) RawHttp() *rawhttp.Client { + return d.rawHttpClient +} + +func (d *Dialers) Close() { + if d.fastDialer != nil { + d.fastDialer.Close() + } + if d.rdapClient.HTTP != nil { + d.rdapClient.HTTP.CloseIdleConnections() + } } // todo: centralizing components is not feasible with current clogged architecture diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 49d612b882..3517cf2f4b 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -23,7 +23,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -113,12 +112,8 @@ func (request *Request) CanCluster(other *Request) bool { // Compile compiles the request generators preparing any requests possible. func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.options = options + request.dialer = options.Dialers.Default() - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errorutil.NewWithTag("ssl", "could not get network client").Wrap(err) - } - request.dialer = client switch { //validate scanmode case request.ScanMode == "": @@ -153,7 +148,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { WildcardCertCheck: true, Retries: request.options.Options.Retries, Timeout: request.options.Options.Timeout, - Fastdialer: client, + Fastdialer: options.Dialers.Default(), ClientHello: true, ServerHello: true, DisplayDns: true, diff --git a/pkg/protocols/ssl/ssl_test.go b/pkg/protocols/ssl/ssl_test.go index 59c9f85f3e..f57f1d9a40 100644 --- a/pkg/protocols/ssl/ssl_test.go +++ b/pkg/protocols/ssl/ssl_test.go @@ -21,11 +21,13 @@ func TestSSLProtocol(t *testing.T) { request := &Request{ Address: "{{Hostname}}", } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) - err := request.Compile(executerOpts) + } + executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo) + require.Nil(t, err, "could not create executer options") + err = request.Compile(executerOpts) require.Nil(t, err, "could not compile ssl request") var gotEvent output.InternalEvent diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index a956de0e3f..9144b9e04b 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -28,7 +28,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -100,13 +99,10 @@ const ( func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.options = options - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errors.Wrap(err, "could not get network client") - } - request.dialer = client + request.dialer = options.Dialers.Default() if len(request.Payloads) > 0 { + var err error request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions()) if err != nil { return errors.Wrap(err, "could not parse payloads") diff --git a/pkg/protocols/whois/rdapclientpool/clientpool.go b/pkg/protocols/whois/rdapclientpool/clientpool.go deleted file mode 100644 index cb393a5055..0000000000 --- a/pkg/protocols/whois/rdapclientpool/clientpool.go +++ /dev/null @@ -1,38 +0,0 @@ -package rdapclientpool - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/types" - "github.com/projectdiscovery/rdap" -) - -var normalClient *rdap.Client - -// Init initializes the client pool implementation -func Init(options *types.Options) error { - // Don't create clients if already created in the past. - if normalClient != nil { - return nil - } - - normalClient = &rdap.Client{} - if options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse { - normalClient.Verbose = func(text string) { - gologger.Debug().Msgf("rdap: %s", text) - } - } - return nil -} - -// Configuration contains the custom configuration options for a client - placeholder -type Configuration struct{} - -// Hash returns the hash of the configuration to allow client pooling - placeholder -func (c *Configuration) Hash() string { - return "" -} - -// Get creates or gets a client for the protocol based on custom configuration -func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) { - return normalClient, nil -} diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index ce0bafe56c..cf3b9d8061 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -22,7 +22,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -64,7 +63,8 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.options = options - request.client, _ = rdapclientpool.Get(options.Options, nil) + + request.client = options.Dialers.Rdap() if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators diff --git a/pkg/reporting/exporters/es/elasticsearch.go b/pkg/reporting/exporters/es/elasticsearch.go index 7b627492c9..8225d8a216 100644 --- a/pkg/reporting/exporters/es/elasticsearch.go +++ b/pkg/reporting/exporters/es/elasticsearch.go @@ -2,7 +2,6 @@ package es import ( "bytes" - "crypto/tls" "encoding/base64" "encoding/json" "fmt" @@ -13,7 +12,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/useragent" ) @@ -56,20 +54,9 @@ type Exporter struct { func New(option *Options) (*Exporter, error) { var ei *Exporter - var client *http.Client - if option.HttpClient != nil { - client = option.HttpClient.HTTPClient - } else { - client = &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 10, - MaxIdleConnsPerHost: 10, - DialContext: protocolstate.Dialer.Dial, - DialTLSContext: protocolstate.Dialer.DialTLS, - TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, - }, - } + // we do not allow nil http client + if option.HttpClient == nil { + return nil, errors.New("http client is nil") } // preparing url for elasticsearch @@ -98,7 +85,7 @@ func New(option *Options) (*Exporter, error) { ei = &Exporter{ url: url, authentication: authentication, - elasticsearch: client, + elasticsearch: option.HttpClient.HTTPClient, } return ei, nil } @@ -131,8 +118,8 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { if err != nil { return err } - defer res.Body.Close() - + defer res.Body.Close() + b, err = io.ReadAll(res.Body) if err != nil { return errors.New(err.Error() + "error thrown by elasticsearch " + string(b)) diff --git a/pkg/reporting/exporters/splunk/splunkhec.go b/pkg/reporting/exporters/splunk/splunkhec.go index e448d6d2fc..d24c960ebb 100644 --- a/pkg/reporting/exporters/splunk/splunkhec.go +++ b/pkg/reporting/exporters/splunk/splunkhec.go @@ -2,17 +2,14 @@ package splunk import ( "bytes" - "crypto/tls" "encoding/json" "fmt" "io" "net" "net/http" - "time" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/useragent" ) @@ -48,20 +45,9 @@ type Exporter struct { func New(option *Options) (*Exporter, error) { var ei *Exporter - var client *http.Client - if option.HttpClient != nil { - client = option.HttpClient.HTTPClient - } else { - client = &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 10, - MaxIdleConnsPerHost: 10, - DialContext: protocolstate.Dialer.Dial, - DialTLSContext: protocolstate.Dialer.DialTLS, - TLSClientConfig: &tls.Config{InsecureSkipVerify: option.SSLVerification}, - }, - } + // we do not allow nil http client + if option.HttpClient == nil { + return nil, errors.New("http client is nil") } // preparing url for splunk @@ -85,7 +71,7 @@ func New(option *Options) (*Exporter, error) { ei = &Exporter{ url: url, authentication: authentication, - splunk: client, + splunk: option.HttpClient.HTTPClient, } return ei, nil } diff --git a/pkg/templates/cluster_test.go b/pkg/templates/cluster_test.go index c8e68b3e07..c6fa356ff2 100644 --- a/pkg/templates/cluster_test.go +++ b/pkg/templates/cluster_test.go @@ -13,10 +13,12 @@ import ( func TestClusterTemplates(t *testing.T) { // state of whether template is flow or multiprotocol is stored in executerOptions i.e why we need to pass it - execOptions := testutils.NewMockExecuterOptions(testutils.DefaultOptions, &testutils.TemplateInfo{ + templateInfo := &testutils.TemplateInfo{ ID: "templateID", Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, - }) + } + execOptions, err := testutils.NewMockExecuterOptions(testutils.DefaultOptions, templateInfo) + require.Nil(t, err, "could not create executer options") t.Run("http-cluster-get", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} tp2 := &Template{Path: "second.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} diff --git a/pkg/testutils/fuzzplayground/server.go b/pkg/testutils/fuzzplayground/server.go index af42552cbf..f0029fcb15 100644 --- a/pkg/testutils/fuzzplayground/server.go +++ b/pkg/testutils/fuzzplayground/server.go @@ -18,7 +18,6 @@ import ( func GetPlaygroundServer() *echo.Echo { e := echo.New() - e.Use(middleware.Recover()) e.Use(middleware.Logger()) e.GET("/", indexHandler) diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 6e6f94d9eb..20c7fdbb8c 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -87,8 +87,12 @@ type TemplateInfo struct { } // NewMockExecuterOptions creates a new mock executeroptions struct -func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions { +func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) (*protocols.ExecutorOptions, error) { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) + dialers, err := protocols.NewDialers(options) + if err != nil { + return nil, err + } executerOpts := &protocols.ExecutorOptions{ TemplateID: info.ID, TemplateInfo: info.Info, @@ -101,9 +105,10 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco Browser: nil, Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Dialers: dialers, } executerOpts.CreateTemplateCtxStore() - return executerOpts + return executerOpts, nil } // NoopWriter is a NooP gologger writer. diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 6a1813efd9..0cc32f5bd3 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -201,11 +201,6 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { } }() - defer func() { - if r := recover(); r != nil { - f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r)) - } - }() if ctx.OnResult == nil { return fmt.Errorf("output callback cannot be nil") diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index b47b38a2a0..cef962a188 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -26,6 +26,11 @@ func setup() { testutils.Init(options) progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) + dialers, err := protocols.NewDialers(options) + if err != nil { + log.Fatalf("Could not create dialers: %s\n", err) + } + executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, @@ -36,6 +41,7 @@ func setup() { Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), Parser: templates.NewParser(), + Dialers: dialers, } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index 4f2aa25e4e..bb47e0adad 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -2,7 +2,6 @@ package multiproto_test import ( "context" - "log" "testing" "time" @@ -21,11 +20,14 @@ import ( var executerOpts protocols.ExecutorOptions -func setup() { +func setup(t *testing.T) { options := testutils.DefaultOptions testutils.Init(options) progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) + dialers, err := protocols.NewDialers(options) + require.Nil(t, err, "could not create dialers") + executerOpts = protocols.ExecutorOptions{ Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, @@ -36,16 +38,16 @@ func setup() { Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), Parser: templates.NewParser(), + Dialers: dialers, } workflowLoader, err := workflow.NewLoader(&executerOpts) - if err != nil { - log.Fatalf("Could not create workflow loader: %s\n", err) - } + require.Nil(t, err, "could not create workflow loader") executerOpts.WorkflowLoader = workflowLoader } func TestMultiProtoWithDynamicExtractor(t *testing.T) { - setup() + setup(t) + Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") @@ -62,7 +64,8 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) { } func TestMultiProtoWithProtoPrefix(t *testing.T) { - setup() + setup(t) + Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") diff --git a/pkg/types/types.go b/pkg/types/types.go index f49da194cc..b9ef397524 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -428,6 +428,10 @@ func (options *Options) HasClientCertificates() bool { return options.ClientCertFile != "" || options.ClientCAFile != "" || options.ClientKeyFile != "" } +func (options *Options) HasDebug() bool { + return options.Verbose || options.Debug || options.DebugRequests || options.DebugResponse +} + // DefaultOptions returns default options for nuclei func DefaultOptions() *Options { return &Options{