diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index a90c4ed52e..bad23f6a46 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -300,6 +300,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"), flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"), flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), + flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), ) flagSet.CreateGroup("interactsh", "interactsh", diff --git a/internal/httpapi/apiendpoint.go b/internal/httpapi/apiendpoint.go new file mode 100644 index 0000000000..2cd055eeea --- /dev/null +++ b/internal/httpapi/apiendpoint.go @@ -0,0 +1,112 @@ +package httpapi + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" + "github.com/projectdiscovery/nuclei/v3/pkg/types" +) + +type Concurrency struct { + BulkSize int `json:"bulk_size"` + Threads int `json:"threads"` + RateLimit int `json:"rate_limit"` + RateLimitDuration string `json:"rate_limit_duration"` + PayloadConcurrency int `json:"payload_concurrency"` + ProbeConcurrency int `json:"probe_concurrency"` + JavascriptConcurrency int `json:"javascript_concurrency"` +} + +// Server represents the HTTP server that handles the concurrency settings endpoints. +type Server struct { + addr string + config *types.Options +} + +// New creates a new instance of Server. +func New(addr string, config *types.Options) *Server { + return &Server{ + addr: addr, + config: config, + } +} + +// Start initializes the server and its routes, then starts listening on the specified address. +func (s *Server) Start() error { + http.HandleFunc("/api/concurrency", s.handleConcurrency) + if err := http.ListenAndServe(s.addr, nil); err != nil { + return err + } + return nil +} + +// handleConcurrency routes the request based on its method to the appropriate handler. +func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + s.getSettings(w, r) + case http.MethodPut: + s.updateSettings(w, r) + default: + http.Error(w, "Unsupported HTTP method", http.StatusMethodNotAllowed) + } +} + +// GetSettings handles GET requests and returns the current concurrency settings +func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) { + concurrencySettings := Concurrency{ + BulkSize: s.config.BulkSize, + Threads: s.config.TemplateThreads, + RateLimit: s.config.RateLimit, + RateLimitDuration: s.config.RateLimitDuration.String(), + PayloadConcurrency: s.config.PayloadConcurrency, + ProbeConcurrency: s.config.ProbeConcurrency, + JavascriptConcurrency: compiler.PoolingJsVmConcurrency, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(concurrencySettings); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +// UpdateSettings handles PUT requests to update the concurrency settings +func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) { + var newSettings Concurrency + if err := json.NewDecoder(r.Body).Decode(&newSettings); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if newSettings.RateLimitDuration != "" { + if duration, err := time.ParseDuration(newSettings.RateLimitDuration); err == nil { + s.config.RateLimitDuration = duration + } else { + http.Error(w, "Invalid duration format", http.StatusBadRequest) + return + } + } + if newSettings.BulkSize > 0 { + s.config.BulkSize = newSettings.BulkSize + } + if newSettings.Threads > 0 { + s.config.TemplateThreads = newSettings.Threads + } + if newSettings.RateLimit > 0 { + s.config.RateLimit = newSettings.RateLimit + } + if newSettings.PayloadConcurrency > 0 { + s.config.PayloadConcurrency = newSettings.PayloadConcurrency + } + if newSettings.ProbeConcurrency > 0 { + s.config.ProbeConcurrency = newSettings.ProbeConcurrency + } + if newSettings.JavascriptConcurrency > 0 { + compiler.PoolingJsVmConcurrency = newSettings.JavascriptConcurrency + s.config.JsConcurrency = newSettings.JavascriptConcurrency // no-op on speed change + } + + w.WriteHeader(http.StatusOK) +} diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go index e7a7c29fae..c2fc815f9f 100644 --- a/internal/runner/inputs.go +++ b/internal/runner/inputs.go @@ -15,8 +15,6 @@ import ( syncutil "github.com/projectdiscovery/utils/sync" ) -var GlobalProbeBulkSize = 50 - // initializeTemplatesHTTPInput initializes the http form of input // for any loaded http templates if input is in non-standard format. func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { @@ -30,11 +28,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { } gologger.Info().Msgf("Running httpx on input host") - var bulkSize = GlobalProbeBulkSize - if r.options.BulkSize > GlobalProbeBulkSize { - bulkSize = r.options.BulkSize - } - httpxOptions := httpx.DefaultOptions httpxOptions.RetryMax = r.options.Retries httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second @@ -43,10 +36,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { return nil, errors.Wrap(err, "could not create httpx client") } - shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize - // Probe the non-standard URLs and store them in cache - swg, err := syncutil.New(syncutil.WithSize(bulkSize)) + swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize)) if err != nil { return nil, errors.Wrap(err, "could not create adaptive group") } @@ -56,8 +47,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { return true } - if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize { - swg.Resize(GlobalProbeBulkSize) + if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency { + swg.Resize(r.options.ProbeConcurrency) } swg.Add() diff --git a/internal/runner/runner.go b/internal/runner/runner.go index fa9b85bf05..12b3e67946 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -31,6 +31,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/internal/colorizer" + "github.com/projectdiscovery/nuclei/v3/internal/httpapi" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" @@ -87,8 +88,9 @@ type Runner struct { pdcpUploadErrMsg string inputProvider provider.InputProvider //general purpose temporary directory - tmpDir string - parser parser.Parser + tmpDir string + parser parser.Parser + httpApiEndpoint *httpapi.Server } const pprofServerAddress = "127.0.0.1:8086" @@ -220,6 +222,17 @@ func New(options *types.Options) (*Runner, error) { }() } + if options.HttpApiEndpoint != "" { + apiServer := httpapi.New(options.HttpApiEndpoint, options) + gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint) + runner.httpApiEndpoint = apiServer + go func() { + if err := apiServer.Start(); err != nil { + gologger.Error().Msgf("Failed to start API server: %s", err) + } + }() + } + if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates { os.Exit(0) } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index e59aa015e2..dc33047aa8 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -55,6 +55,7 @@ var DefaultOptions = &types.Options{ Retries: 1, RateLimit: 150, RateLimitDuration: time.Second, + ProbeConcurrency: 50, ProjectPath: "", Severities: severity.Severities{}, Targets: []string{}, diff --git a/pkg/types/types.go b/pkg/types/types.go index 81a2731689..2768a70fd1 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -389,6 +389,8 @@ type Options struct { ProbeConcurrency int // Dast only runs DAST templates DAST bool + // HttpApiEndpoint is the experimental http api endpoint + HttpApiEndpoint string } // ShouldLoadResume resume file @@ -420,6 +422,7 @@ func DefaultOptions() *Options { TemplateThreads: 25, HeadlessBulkSize: 10, HeadlessTemplateThreads: 10, + ProbeConcurrency: 50, Timeout: 5, Retries: 1, MaxHostError: 30,