globals => shared struct

cmd/integration-test/library.go

@@ -100,6 +100,12 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	ratelimiter := ratelimit.New(context.Background(), 150, time.Second)
 	defer ratelimiter.Stop()
 
+	dialers, err := protocols.NewDialers(defaultOpts)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create dialers")
+	}
+	defer dialers.Close()
+
 	executerOpts := protocols.ExecutorOptions{
 		Output:          outputWriter,
 		Options:         defaultOpts,
@@ -112,6 +118,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 		Colorizer:       aurora.NewAurora(true),
 		ResumeCfg:       types.NewResumeCfg(),
 		Parser:          templates.NewParser(),
+		Dialers:         dialers,
 	}
 	engine := core.New(defaultOpts)
 	engine.SetExecuterOptions(executerOpts)

internal/runner/options.go

@@ -32,7 +32,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	fileutil "github.com/projectdiscovery/utils/file"
 	"github.com/projectdiscovery/utils/generic"
-	logutil "github.com/projectdiscovery/utils/log"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
@@ -342,7 +341,7 @@ func configureOutput(options *types.Options) {
 	}
 
 	// disable standard logger (ref: https://github.com/golang/go/issues/19895)
-	logutil.DisableDefaultLogger()
+	// logutil.DisableDefaultLogger()
 }
 
 // loadResolvers loads resolvers from both user-provided flags and file

internal/runner/runner.go

@@ -183,6 +183,7 @@ func New(options *types.Options) (*Runner, error) {
 	var httpclient *retryablehttp.Client
 	if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
 		var err error
+		// we use an independent instance of retryablehttp for internal operations
 		httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{})
 		if err != nil {
 			return nil, err
@@ -453,6 +454,11 @@ func (r *Runner) RunEnumeration() error {
 	fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency)
 	r.fuzzFrequencyCache = fuzzFreqCache
 
+	dialers, err := protocols.NewDialers(r.options)
+	if err != nil {
+		return errors.Wrap(err, "could not create dialers")
+	}
+
 	// Create the executor options which will be used throughout the execution
 	// stage by the nuclei engine modules.
 	executorOpts := protocols.ExecutorOptions{
@@ -472,6 +478,7 @@ func (r *Runner) RunEnumeration() error {
 		TemporaryDirectory:  r.tmpDir,
 		Parser:              r.parser,
 		FuzzParamsFrequency: fuzzFreqCache,
+		Dialers:             dialers,
 	}
 
 	if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {

lib/multi.go

@@ -40,6 +40,7 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
 		Colorizer:       aurora.NewAurora(true),
 		ResumeCfg:       types.NewResumeCfg(),
 		Parser:          base.parser,
+		Dialers:         base.dialers,
 	}
 	if opts.RateLimitMinute > 0 {
 		opts.RateLimit = opts.RateLimitMinute

lib/sdk.go

@@ -76,6 +76,7 @@ type NucleiEngine struct {
 	httpClient       *retryablehttp.Client
 	parser           *templates.Parser
 	authprovider     authprovider.AuthProvider
+	dialers          *protocols.Dialers
 
 	// unexported meta options
 	opts           *types.Options
@@ -214,6 +215,12 @@ func (e *NucleiEngine) closeInternal() {
 	if e.httpxClient != nil {
 		_ = e.httpxClient.Close()
 	}
+	if e.dialers != nil {
+		e.executerOpts.Dialers.Close()
+	}
+	if e.executerOpts.Dialers != nil {
+		e.executerOpts.Dialers.Close()
+	}
 }
 
 // Close all resources used by nuclei engine

lib/sdk_private.go

@@ -108,7 +108,14 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		return err
 	}
 
+	var err error
+	e.dialers, err = protocols.NewDialers(e.opts)
+	if err != nil {
+		return errors.Wrap(err, "could not create dialers")
+	}
+
 	if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
+		// we use an independent instance of retryablehttp for internal operations
 		httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
 		if err != nil {
 			return err
@@ -127,7 +134,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 	})
 
 	e.applyRequiredDefaults(ctx)
-	var err error
 
 	// setup progressbar
 	if e.enableStats {
@@ -171,6 +177,7 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		ResumeCfg:       types.NewResumeCfg(),
 		Browser:         e.browserInstance,
 		Parser:          e.parser,
+		Dialers:         e.dialers,
 	}
 	if len(e.opts.SecretsFile) > 0 {
 		authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts)

pkg/js/compiler/compiler.go

@@ -117,11 +117,6 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 	defer cancel()
 	// execute the script
 	results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
-		defer func() {
-			if r := recover(); r != nil {
-				err = fmt.Errorf("panic: %v", r)
-			}
-		}()
 		return ExecuteProgram(program, args, opts)
 	})
 	if err != nil {

pkg/js/compiler/pool.go

@@ -84,11 +84,6 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg
 			opts.Cleanup(runtime)
 		}
 	}()
-	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("panic: %s", r)
-		}
-	}()
 	// set template ctx
 	_ = runtime.Set("template", args.TemplateCtx)
 	// set args

pkg/operators/common/dsl/dsl.go

@@ -61,11 +61,7 @@ func init() {
 			return nil, fmt.Errorf("invalid dns type")
 		}
 
-		err := dnsclientpool.Init(&types.Options{})
-		if err != nil {
-			return nil, err
-		}
-		dnsClient, err := dnsclientpool.Get(nil, &dnsclientpool.Configuration{})
+		dnsClient, err := dnsclientpool.Get(&types.Options{}, &dnsclientpool.Configuration{Retries: 11})
 		if err != nil {
 			return nil, err
 		}

pkg/protocols/code/code_test.go

@@ -24,11 +24,14 @@ func TestCodeProtocol(t *testing.T) {
 		Engine: []string{"sh"},
 		Source: "echo test",
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+	templateInfo := &testutils.TemplateInfo{
 		ID:   templateID,
 		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
-	err := request.Compile(executerOpts)
+	}
+	executerOpts, err := testutils.NewMockExecuterOptions(options, templateInfo)
+	require.Nil(t, err, "could not create executer options")
+
+	err = request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile code request")
 
 	var gotEvent output.InternalEvent

pkg/protocols/common/automaticscan/automaticscan.go

@@ -95,22 +95,29 @@ func New(opts Options) (*Service, error) {
 		return nil, err
 	}
 
-	httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
+	httpConnectionOptions := &httpclientpool.Configuration{
 		Connection: &httpclientpool.ConnectionConfiguration{
 			DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options),
 		},
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get http client")
 	}
+	var httpClient *retryablehttp.Client
+	if httpConnectionOptions.HasStandardOptions() {
+		httpClient = opts.ExecuterOpts.Dialers.Http()
+	} else {
+		httpClient, err = httpclientpool.Get(opts.ExecuterOpts.Options, httpConnectionOptions)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get http client")
+		}
+	}
+
 	return &Service{
 		opts:               opts.ExecuterOpts,
 		store:              opts.Store,
 		engine:             opts.Engine,
 		target:             opts.Target,
 		wappalyzer:         wappalyzer,
 		templateDirs:       templateDirs, // fix this
-		httpclient:         httpclient,
+		httpclient:         httpClient,
 		technologyMappings: mappingData,
 		techTemplates:      techDetectTemplates,
 		ServiceOpts:        opts,

pkg/protocols/common/protocolinit/init.go

@@ -3,11 +3,8 @@ package protocolinit
 import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
@@ -16,21 +13,12 @@ func Init(options *types.Options) error {
 	if err := protocolstate.Init(options); err != nil {
 		return err
 	}
-	if err := dnsclientpool.Init(options); err != nil {
-		return err
-	}
 	if err := httpclientpool.Init(options); err != nil {
 		return err
 	}
 	if err := signerpool.Init(options); err != nil {
 		return err
 	}
-	if err := networkclientpool.Init(options); err != nil {
-		return err
-	}
-	if err := rdapclientpool.Init(options); err != nil {
-		return err
-	}
 	if err := compiler.Init(options); err != nil {
 		return err
 	}

pkg/protocols/common/protocolstate/state.go

@@ -26,12 +26,7 @@ func ShouldInit() bool {
 	return Dialer == nil
 }
 
-// Init creates the Dialer instance based on user configuration
-func Init(options *types.Options) error {
-	if Dialer != nil {
-		return nil
-	}
-
+func GetDialerFromOptions(options *types.Options) (*fastdialer.Dialer, error) {
 	lfaAllowed = options.AllowLocalFileAccess
 	opts := fastdialer.DefaultOptions
 	if options.DialerTimeout > 0 {
@@ -66,7 +61,7 @@ func Init(options *types.Options) error {
 	case options.SourceIP != "" && options.Interface != "":
 		isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, options.Interface)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		if isAssociated {
 			opts.Dialer = &net.Dialer{
@@ -75,12 +70,12 @@ func Init(options *types.Options) error {
 				},
 			}
 		} else {
-			return fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface)
+			return nil, fmt.Errorf("source ip (%s) is not associated with the interface (%s)", options.SourceIP, options.Interface)
 		}
 	case options.SourceIP != "":
 		isAssociated, err := isIpAssociatedWithInterface(options.SourceIP, "any")
 		if err != nil {
-			return err
+			return nil, err
 		}
 		if isAssociated {
 			opts.Dialer = &net.Dialer{
@@ -89,12 +84,12 @@ func Init(options *types.Options) error {
 				},
 			}
 		} else {
-			return fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP)
+			return nil, fmt.Errorf("source ip (%s) is not associated with any network interface", options.SourceIP)
 		}
 	case options.Interface != "":
 		ifadrr, err := interfaceAddress(options.Interface)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		opts.Dialer = &net.Dialer{
 			LocalAddr: &net.TCPAddr{
@@ -105,7 +100,7 @@ func Init(options *types.Options) error {
 	if types.ProxySocksURL != "" {
 		proxyURL, err := url.Parse(types.ProxySocksURL)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		var forward *net.Dialer
 		if opts.Dialer != nil {
@@ -119,7 +114,7 @@ func Init(options *types.Options) error {
 		}
 		dialer, err := proxy.FromURL(proxyURL, forward)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		opts.ProxyDialer = &dialer
 	}
@@ -143,9 +138,26 @@ func Init(options *types.Options) error {
 	// fastdialer now by default fallbacks to ztls when there are tls related errors
 	dialer, err := fastdialer.NewDialer(opts)
 	if err != nil {
-		return errors.Wrap(err, "could not create dialer")
+		return nil, errors.Wrap(err, "could not create dialer")
+	}
+
+	return dialer, nil
+}
+
+// Init creates the Dialer instance based on user configuration
+func Init(options *types.Options) error {
+	// ***************************
+	// todo: remove this part
+	if Dialer != nil {
+		return nil
+	}
+
+	dialer, err := GetDialerFromOptions(options)
+	if err != nil {
+		return err
 	}
 	Dialer = dialer
+	// ********************+*****
 
 	// override dialer in mysql
 	mysql.RegisterDialContext("tcp", func(ctx context.Context, addr string) (net.Conn, error) {

pkg/protocols/dns/dns.go

@@ -189,25 +189,26 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 }
 
 func (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {
-	dnsClientOptions := &dnsclientpool.Configuration{
-		Retries: request.Retries,
-	}
-	if len(request.Resolvers) > 0 {
-		if len(request.Resolvers) > 0 {
-			for _, resolver := range request.Resolvers {
-				if expressions.ContainsUnresolvedVariables(resolver) != nil {
-					var err error
-					resolver, err = expressions.Evaluate(resolver, metadata)
-					if err != nil {
-						return nil, errors.Wrap(err, "could not resolve resolvers expressions")
-					}
-					dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver)
+	// if retries are set or resolvers are set, we need to create a new client
+	if request.Retries > 0 || len(request.Resolvers) > 0 {
+		dnsClientOptions := &dnsclientpool.Configuration{
+			Retries: request.Retries,
+		}
+		for _, resolver := range request.Resolvers {
+			if expressions.ContainsUnresolvedVariables(resolver) != nil {
+				var err error
+				resolver, err = expressions.Evaluate(resolver, metadata)
+				if err != nil {
+					return nil, errors.Wrap(err, "could not resolve resolvers expressions")
 				}
+				dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver)
 			}
 		}
 		dnsClientOptions.Resolvers = request.Resolvers
+		return dnsclientpool.Get(options.Options, dnsClientOptions)
 	}
-	return dnsclientpool.Get(options.Options, dnsClientOptions)
+	// otherwise we return the default dns client
+	return options.Dialers.Dns(), nil
 }
 
 // Requests returns the total number of requests the YAML rule will perform