Exposing embedded api for settings control in CLI modality

cmd/nuclei/main.go

@@ -300,6 +300,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
+		flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
 	)
 
 	flagSet.CreateGroup("interactsh", "interactsh",

internal/httpapi/apiendpoint.go

@@ -0,0 +1,112 @@
+package httpapi
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
+
+type Concurrency struct {
+	BulkSize              int    `json:"bulk_size"`
+	Threads               int    `json:"threads"`
+	RateLimit             int    `json:"rate_limit"`
+	RateLimitDuration     string `json:"rate_limit_duration"`
+	PayloadConcurrency    int    `json:"payload_concurrency"`
+	ProbeConcurrency      int    `json:"probe_concurrency"`
+	JavascriptConcurrency int    `json:"javascript_concurrency"`
+}
+
+// Server represents the HTTP server that handles the concurrency settings endpoints.
+type Server struct {
+	addr   string
+	config *types.Options
+}
+
+// New creates a new instance of Server.
+func New(addr string, config *types.Options) *Server {
+	return &Server{
+		addr:   addr,
+		config: config,
+	}
+}
+
+// Start initializes the server and its routes, then starts listening on the specified address.
+func (s *Server) Start() error {
+	http.HandleFunc("/api/concurrency", s.handleConcurrency)
+	if err := http.ListenAndServe(s.addr, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+// handleConcurrency routes the request based on its method to the appropriate handler.
+func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		s.getSettings(w, r)
+	case http.MethodPut:
+		s.updateSettings(w, r)
+	default:
+		http.Error(w, "Unsupported HTTP method", http.StatusMethodNotAllowed)
+	}
+}
+
+// GetSettings handles GET requests and returns the current concurrency settings
+func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) {
+	concurrencySettings := Concurrency{
+		BulkSize:              s.config.BulkSize,
+		Threads:               s.config.TemplateThreads,
+		RateLimit:             s.config.RateLimit,
+		RateLimitDuration:     s.config.RateLimitDuration.String(),
+		PayloadConcurrency:    s.config.PayloadConcurrency,
+		ProbeConcurrency:      s.config.ProbeConcurrency,
+		JavascriptConcurrency: compiler.PoolingJsVmConcurrency,
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(concurrencySettings); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// UpdateSettings handles PUT requests to update the concurrency settings
+func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) {
+	var newSettings Concurrency
+	if err := json.NewDecoder(r.Body).Decode(&newSettings); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if newSettings.RateLimitDuration != "" {
+		if duration, err := time.ParseDuration(newSettings.RateLimitDuration); err == nil {
+			s.config.RateLimitDuration = duration
+		} else {
+			http.Error(w, "Invalid duration format", http.StatusBadRequest)
+			return
+		}
+	}
+	if newSettings.BulkSize > 0 {
+		s.config.BulkSize = newSettings.BulkSize
+	}
+	if newSettings.Threads > 0 {
+		s.config.TemplateThreads = newSettings.Threads
+	}
+	if newSettings.RateLimit > 0 {
+		s.config.RateLimit = newSettings.RateLimit
+	}
+	if newSettings.PayloadConcurrency > 0 {
+		s.config.PayloadConcurrency = newSettings.PayloadConcurrency
+	}
+	if newSettings.ProbeConcurrency > 0 {
+		s.config.ProbeConcurrency = newSettings.ProbeConcurrency
+	}
+	if newSettings.JavascriptConcurrency > 0 {
+		compiler.PoolingJsVmConcurrency = newSettings.JavascriptConcurrency
+		s.config.JsConcurrency = newSettings.JavascriptConcurrency // no-op on speed change
+	}
+
+	w.WriteHeader(http.StatusOK)
+}

internal/runner/inputs.go

@@ -15,8 +15,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var GlobalProbeBulkSize = 50
-
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
@@ -30,11 +28,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = GlobalProbeBulkSize
-	if r.options.BulkSize > GlobalProbeBulkSize {
-		bulkSize = r.options.BulkSize
-	}
-
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -43,10 +36,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
-	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
-
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -56,8 +47,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
-		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
-			swg.Resize(GlobalProbeBulkSize)
+		if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency {
+			swg.Resize(r.options.ProbeConcurrency)
 		}
 
 		swg.Add()

internal/runner/runner.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v3/internal/httpapi"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
@@ -87,8 +88,9 @@ type Runner struct {
 	pdcpUploadErrMsg string
 	inputProvider    provider.InputProvider
 	//general purpose temporary directory
-	tmpDir string
-	parser parser.Parser
+	tmpDir          string
+	parser          parser.Parser
+	httpApiEndpoint *httpapi.Server
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -220,6 +222,17 @@ func New(options *types.Options) (*Runner, error) {
 		}()
 	}
 
+	if options.HttpApiEndpoint != "" {
+		apiServer := httpapi.New(options.HttpApiEndpoint, options)
+		gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint)
+		runner.httpApiEndpoint = apiServer
+		go func() {
+			if err := apiServer.Start(); err != nil {
+				gologger.Error().Msgf("Failed to start API server: %s", err)
+			}
+		}()
+	}
+
 	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}

pkg/testutils/testutils.go

@@ -55,6 +55,7 @@ var DefaultOptions = &types.Options{
 	Retries:                    1,
 	RateLimit:                  150,
 	RateLimitDuration:          time.Second,
+	ProbeConcurrency:           50,
 	ProjectPath:                "",
 	Severities:                 severity.Severities{},
 	Targets:                    []string{},

pkg/types/types.go

@@ -389,6 +389,8 @@ type Options struct {
 	ProbeConcurrency int
 	// Dast only runs DAST templates
 	DAST bool
+	// HttpApiEndpoint is the experimental http api endpoint
+	HttpApiEndpoint string
 }
 
 // ShouldLoadResume resume file
@@ -420,6 +422,7 @@ func DefaultOptions() *Options {
 		TemplateThreads:         25,
 		HeadlessBulkSize:        10,
 		HeadlessTemplateThreads: 10,
+		ProbeConcurrency:        50,
 		Timeout:                 5,
 		Retries:                 1,
 		MaxHostError:            30,