From ea2e13a4aa83ac349e3fde29bdbaa0814a2540f9 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:57:32 +0530 Subject: [PATCH] nuclei 'stats' build : scan events + chart utils (#5032) * prototype new scan events * scan-event: improvements + conditional build * add scan charts server: make scan-charts * scan-charts: bug fix --- .gitignore | 3 + Makefile | 9 + cmd/scan-charts/main.go | 40 ++++ go.mod | 3 + go.sum | 4 + internal/runner/runner.go | 15 ++ pkg/scan/charts/charts.go | 87 ++++++++ pkg/scan/charts/echarts.go | 351 +++++++++++++++++++++++++++++++++ pkg/scan/events/scan_noop.go | 11 ++ pkg/scan/events/stats_build.go | 80 ++++++++ pkg/scan/events/utils.go | 45 +++++ pkg/scan/scan_context.go | 1 + pkg/tmplexec/exec.go | 41 ++++ pkg/types/types.go | 2 + 14 files changed, 692 insertions(+) create mode 100644 cmd/scan-charts/main.go create mode 100644 pkg/scan/charts/charts.go create mode 100644 pkg/scan/charts/echarts.go create mode 100644 pkg/scan/events/scan_noop.go create mode 100644 pkg/scan/events/stats_build.go create mode 100644 pkg/scan/events/utils.go diff --git a/.gitignore b/.gitignore index 65815d8db6..9386137b54 100644 --- a/.gitignore +++ b/.gitignore @@ -36,5 +36,8 @@ pkg/protocols/headless/engine/.cache /fuzzplayground integration_tests/fuzzplayground /dsl.md +/nuclei-stats +/nuclei-stats-* +/scan-charts diff --git a/Makefile b/Makefile index dadbe6205d..e916b980a6 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,18 @@ ifneq ($(shell go env GOOS),darwin) LDFLAGS := -extldflags "-static" endif +.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs + all: build build: + rm -f nuclei 2>/dev/null $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go +build-stats: + rm -f nuclei-stats 2>/dev/null + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go +scan-charts: + rm -f scan-charts 2>/dev/null + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go docs: if ! which dstdocgen > /dev/null; then echo -e "Command not found! Install? (y/n) \c" diff --git a/cmd/scan-charts/main.go b/cmd/scan-charts/main.go new file mode 100644 index 0000000000..644a1f004e --- /dev/null +++ b/cmd/scan-charts/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "flag" + + "github.com/projectdiscovery/nuclei/v3/pkg/scan/charts" +) + +var ( + dir string + address string + output string +) + +func main() { + flag.StringVar(&dir, "dir", "", "directory to scan") + flag.StringVar(&address, "address", ":9000", "address to run the server on") + flag.StringVar(&output, "output", "", "output filename of generated html file") + flag.Parse() + + if dir == "" { + flag.Usage() + return + } + + server, err := charts.NewScanEventsCharts(dir) + if err != nil { + panic(err) + } + server.PrintInfo() + + if output != "" { + if err = server.GenerateHTML(output); err != nil { + panic(err) + } + return + } + + server.Start(address) +} diff --git a/go.mod b/go.mod index a533740474..4cf94ec00c 100644 --- a/go.mod +++ b/go.mod @@ -329,6 +329,7 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562 github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-echarts/go-echarts/v2 v2.3.3 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect @@ -348,3 +349,5 @@ require ( // https://go.dev/ref/mod#go-mod-file-retract retract v3.2.0 // retract due to broken js protocol issue + +replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 diff --git a/go.sum b/go.sum index 290fd9e482..888602a714 100644 --- a/go.sum +++ b/go.sum @@ -341,6 +341,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg= +github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -1013,6 +1015,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo= +github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index fa9b85bf05..174c06f0f7 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" "github.com/projectdiscovery/nuclei/v3/pkg/installer" "github.com/projectdiscovery/nuclei/v3/pkg/loader/parser" + "github.com/projectdiscovery/nuclei/v3/pkg/scan/events" uncoverlib "github.com/projectdiscovery/uncover" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" @@ -553,6 +554,20 @@ func (r *Runner) RunEnumeration() error { executorOpts.InputHelper.InputsHTTP = inputHelpers } + // initialize stats worker ( this is no-op unless nuclei is built with stats build tag) + // during execution a directory with 2 files will be created in the current directory + // config.json - containing below info + // events.jsonl - containing all start and end times of all templates + events.InitWithConfig(&events.ScanConfig{ + Name: "nuclei-stats", // make this configurable + TargetCount: int(r.inputProvider.Count()), + TemplatesCount: len(store.Templates()) + len(store.Workflows()), + TemplateConcurrency: r.options.TemplateThreads, + PayloadConcurrency: r.options.PayloadConcurrency, + JsConcurrency: r.options.JsConcurrency, + Retries: r.options.Retries, + }, "") + enumeration := false var results *atomic.Bool results, err = r.runStandardEnumeration(executorOpts, store, executorEngine) diff --git a/pkg/scan/charts/charts.go b/pkg/scan/charts/charts.go new file mode 100644 index 0000000000..03bfdcc6c2 --- /dev/null +++ b/pkg/scan/charts/charts.go @@ -0,0 +1,87 @@ +package charts + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/labstack/echo/v4" + "github.com/projectdiscovery/nuclei/v3/pkg/scan/events" + fileutil "github.com/projectdiscovery/utils/file" +) + +// ScanEventsCharts is a struct for nuclei event charts +type ScanEventsCharts struct { + eventsDir string + config *events.ScanConfig + data []events.ScanEvent +} + +func (sc *ScanEventsCharts) PrintInfo() { + fmt.Printf("[+] Scan Info\n") + fmt.Printf(" - Name: %s\n", sc.config.Name) + fmt.Printf(" - Target Count: %d\n", sc.config.TargetCount) + fmt.Printf(" - Template Count: %d\n", sc.config.TemplatesCount) + fmt.Printf(" - Template Concurrency: %d\n", sc.config.TemplateConcurrency) + fmt.Printf(" - Payload Concurrency: %d\n", sc.config.PayloadConcurrency) + fmt.Printf(" - Retries: %v\n", sc.config.Retries) + fmt.Printf(" - Total Events: %d\n", len(sc.data)) + fmt.Println() +} + +// NewScanEventsCharts creates a new nuclei event charts +func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) { + sc := &ScanEventsCharts{eventsDir: eventsDir} + if !fileutil.FolderExists(eventsDir) { + return nil, fmt.Errorf("events directory does not exist") + } + // open two files + // config.json + bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile)) + if err != nil { + return nil, err + } + var config events.ScanConfig + err = json.Unmarshal(bin, &config) + if err != nil { + return nil, err + } + sc.config = &config + + // events.jsonl + f, err := os.Open(filepath.Join(eventsDir, events.EventsFile)) + if err != nil { + return nil, err + } + defer f.Close() + + data := []events.ScanEvent{} + dec := json.NewDecoder(f) + for { + var event events.ScanEvent + if err := dec.Decode(&event); err != nil { + break + } + data = append(data, event) + } + sc.data = data + + if len(data) == 0 { + return nil, fmt.Errorf("no events found in the events file") + } + + return sc, nil +} + +// Start starts the nuclei event charts server +func (sc *ScanEventsCharts) Start(addr string) { + e := echo.New() + e.HideBanner = true + e.GET("/concurrency", sc.ConcurrencyVsTime) + e.GET("/requests", sc.TotalRequestsOverTime) + e.GET("/slow", sc.TopSlowTemplates) + e.GET("/rps", sc.RequestsVSInterval) + e.GET("/", sc.AllCharts) + e.Logger.Fatal(e.Start(addr)) +} diff --git a/pkg/scan/charts/echarts.go b/pkg/scan/charts/echarts.go new file mode 100644 index 0000000000..bf70fee710 --- /dev/null +++ b/pkg/scan/charts/echarts.go @@ -0,0 +1,351 @@ +package charts + +import ( + "fmt" + "os" + "sort" + "time" + + "github.com/go-echarts/go-echarts/v2/charts" + "github.com/go-echarts/go-echarts/v2/components" + "github.com/go-echarts/go-echarts/v2/opts" + "github.com/labstack/echo/v4" + "github.com/projectdiscovery/nuclei/v3/pkg/scan/events" + sliceutil "github.com/projectdiscovery/utils/slice" +) + +const ( + TopK = 50 + SpacerHeight = "50px" +) + +func (s *ScanEventsCharts) AllCharts(c echo.Context) error { + page := s.allCharts(c) + return page.Render(c.Response().Writer) +} + +func (s *ScanEventsCharts) GenerateHTML(filePath string) error { + page := s.allCharts(nil) + output, err := os.Create(filePath) + if err != nil { + return err + } + return page.Render(output) +} + +// AllCharts generates all the charts for the scan events and returns a page component +func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page { + page := components.NewPage() + page.PageTitle = "Nuclei Charts" + line1 := s.totalRequestsOverTime(c) + line1.SetSpacerHeight(SpacerHeight) + kline := s.topSlowTemplates(c) + kline.SetSpacerHeight(SpacerHeight) + line2 := s.requestsVSInterval(c) + line2.SetSpacerHeight(SpacerHeight) + line3 := s.concurrencyVsTime(c) + line3.SetSpacerHeight(SpacerHeight) + page.AddCharts(line1, kline, line2, line3) + page.Validate() + page.SetLayout(components.PageCenterLayout) + return page +} + +func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error { + line := s.totalRequestsOverTime(c) + return line.Render(c.Response().Writer) +} + +// totalRequestsOverTime generates a line chart showing total requests count over time +func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line { + line := charts.NewLine() + line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)") + + var startTime time.Time = time.Now() + var endTime time.Time + + for _, event := range s.data { + if event.Time.Before(startTime) { + startTime = event.Time + } + if event.Time.After(endTime) { + endTime = event.Time + } + } + data := getCategoryRequestCount(s.data) + max := 0 + for _, v := range data { + if len(v) > max { + max = len(v) + } + } + line.SetXAxis(time.Now().Format(time.RFC3339)) + for k, v := range data { + lineData := make([]opts.LineData, 0) + temp := 0 + for _, scanEvent := range v { + temp += scanEvent.MaxRequests + val := scanEvent.Time.Sub(startTime) + lineData = append(lineData, opts.LineData{ + Value: []interface{}{val.Milliseconds(), temp}, + Name: scanEvent.TemplateID, + }) + } + line.AddSeries(k, lineData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"})) + } + + line.SetGlobalOptions( + charts.WithTitleOpts(opts.Title{Title: "Nuclei: total-req vs time"}), + charts.WithXAxisOpts(opts.XAxis{Name: "Time", Type: "time", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}), + charts.WithYAxisOpts(opts.YAxis{Name: "Requests Sent", Type: "value"}), + charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}), + charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}), + charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}), + charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{ + SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"}, + DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}}, + DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}}, + }}), + ) + + line.Validate() + return line +} + +func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error { + kline := s.topSlowTemplates(c) + return kline.Render(c.Response().Writer) +} + +// topSlowTemplates generates a Kline chart showing the top slow templates by time taken +func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline { + kline := charts.NewKLine() + kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK)) + + ids := map[string][]int64{} + var startTime time.Time = time.Now() + for _, event := range s.data { + if event.Time.Before(startTime) { + startTime = event.Time + } + } + for _, event := range s.data { + ids[event.TemplateID] = append(ids[event.TemplateID], event.Time.Sub(startTime).Milliseconds()) + } + + type entry struct { + ID string + KlineData opts.KlineData + start int64 + end int64 + } + data := []entry{} + + for a, b := range ids { + if len(b) < 2 { + continue // Prevents index out of range error + } + d := entry{ + ID: a, + KlineData: opts.KlineData{Value: []int64{b[0], b[len(b)-1], b[0], b[len(b)-1]}}, // Adjusted to prevent index out of range error + start: b[0], + end: b[len(b)-1], + } + data = append(data, d) + } + + sort.Slice(data, func(i, j int) bool { + return data[i].end-data[i].start > data[j].end-data[j].start + }) + + x := make([]string, 0) + y := make([]opts.KlineData, 0) + for _, event := range data[:TopK] { + x = append(x, event.ID) + y = append(y, event.KlineData) + } + + kline.SetXAxis(x).AddSeries("templates", y) + kline.SetGlobalOptions( + charts.WithTitleOpts(opts.Title{Title: fmt.Sprintf("Nuclei: Top %v Slow Templates", TopK)}), + charts.WithXAxisOpts(opts.XAxis{ + Type: "category", + Show: true, + AxisLabel: &opts.AxisLabel{Rotate: 90, Show: true, ShowMinLabel: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (value) { return value; }`)}, + }), + charts.WithYAxisOpts(opts.YAxis{ + Scale: true, + Type: "value", + Show: true, + AxisLabel: &opts.AxisLabel{Show: true, Formatter: opts.FuncOpts(`function (ms) { return Math.floor(ms/60000) + 'm' + Math.floor((ms/60000 - Math.floor(ms/60000))*60) + 's'; }`)}, + }), + charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}), + charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "40%", Top: "10%"}), + charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "events.ScanEvent", TriggerOn: "mousemove|click", Enterable: true, Formatter: opts.FuncOpts(`function (params) { return params.name ; }`)}), + charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{ + SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"}, + DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}}, + DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}}, + }}), + ) + + return kline +} + +func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error { + line := s.requestsVSInterval(c) + return line.Render(c.Response().Writer) +} + +// requestsVSInterval generates a line chart showing requests per second over time +func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line { + line := charts.NewLine() + line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time") + + sort.Slice(s.data, func(i, j int) bool { + return s.data[i].Time.Before(s.data[j].Time) + }) + + var interval time.Duration + + if c != nil { + interval, _ = time.ParseDuration(c.QueryParam("interval")) + } + if interval <= 3 { + interval = 5 * time.Second + } + + data := []opts.LineData{} + temp := 0 + if len(s.data) > 0 { + orig := s.data[0].Time + startTime := orig + xaxisData := []int64{} + for _, v := range s.data { + if v.Time.Sub(startTime) > interval { + millisec := v.Time.Sub(orig).Milliseconds() + xaxisData = append(xaxisData, millisec) + data = append(data, opts.LineData{Value: temp, Name: v.Time.Sub(orig).String()}) + temp = 0 + startTime = v.Time + } + temp += 1 + } + // Handle last interval if exists + if temp > 0 { + millisec := s.data[len(s.data)-1].Time.Sub(orig).Milliseconds() + xaxisData = append(xaxisData, millisec) + data = append(data, opts.LineData{Value: temp, Name: s.data[len(s.data)-1].Time.Sub(orig).String()}) + } + line.SetXAxis(xaxisData) + line.AddSeries("RPS", data, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"})) + } + + line.SetGlobalOptions( + charts.WithTitleOpts(opts.Title{Title: "Nuclei: Template Execution", Subtitle: "Time Interval: " + interval.String()}), + charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}), + charts.WithYAxisOpts(opts.YAxis{Name: "RPS Value", Type: "value", Show: true}), + charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}), + charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}), + charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}), + charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{ + SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"}, + DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}}, + DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}}, + }}), + ) + + line.Validate() + return line +} + +func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error { + line := s.concurrencyVsTime(c) + return line.Render(c.Response().Writer) +} + +// concurrencyVsTime generates a line chart showing concurrency (total workers) over time +func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line { + line := charts.NewLine() + line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time") + + dataset := sliceutil.Clone(s.data) + + sort.Slice(dataset, func(i, j int) bool { + return dataset[i].Time.Before(dataset[j].Time) + }) + + var interval time.Duration + if c != nil { + interval, _ = time.ParseDuration(c.QueryParam("interval")) + } + if interval <= 3 { + interval = 5 * time.Second + } + + // create array with time interval as x-axis and worker count as y-axis + // entry is a struct with time and poolsize + type entry struct { + Time time.Duration + poolsize int + } + allEntries := []entry{} + + dataIndex := 0 + maxIndex := len(dataset) - 1 + currEntry := entry{} + + lastTime := dataset[0].Time + for dataIndex <= maxIndex { + currTime := dataset[dataIndex].Time + if currTime.Sub(lastTime) > interval { + // next batch + currEntry.Time = interval + allEntries = append(allEntries, currEntry) + lastTime = dataset[dataIndex-1].Time + } + if dataset[dataIndex].EventType == events.ScanStarted { + currEntry.poolsize += 1 + } else { + currEntry.poolsize -= 1 + } + dataIndex += 1 + } + + plotData := []opts.LineData{} + xaxisData := []int64{} + tempTime := time.Duration(0) + for _, v := range allEntries { + tempTime += v.Time + plotData = append(plotData, opts.LineData{Value: v.poolsize, Name: tempTime.String()}) + xaxisData = append(xaxisData, tempTime.Milliseconds()) + } + line.SetXAxis(xaxisData) + line.AddSeries("Concurrency", plotData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"})) + + line.SetGlobalOptions( + charts.WithTitleOpts(opts.Title{Title: "Nuclei: WorkerPool", Subtitle: "Time Interval: " + interval.String()}), + charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}), + charts.WithYAxisOpts(opts.YAxis{Name: "Total Workers", Type: "value", Show: true}), + charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}), + charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}), + charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}), + charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{ + SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"}, + DataZoom: &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}}, + DataView: &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}}, + }}), + ) + + line.Validate() + return line +} + +// getCategoryRequestCount returns a map of category and request count +func getCategoryRequestCount(values []events.ScanEvent) map[string][]events.ScanEvent { + mx := make(map[string][]events.ScanEvent) + for _, event := range values { + mx[event.TemplateType] = append(mx[event.TemplateType], event) + } + return mx +} diff --git a/pkg/scan/events/scan_noop.go b/pkg/scan/events/scan_noop.go new file mode 100644 index 0000000000..a284657f52 --- /dev/null +++ b/pkg/scan/events/scan_noop.go @@ -0,0 +1,11 @@ +//go:build !stats +// +build !stats + +package events + +// AddScanEvent is a no-op function +func AddScanEvent(event ScanEvent) { +} + +func InitWithConfig(config *ScanConfig, statsDirectory string) { +} diff --git a/pkg/scan/events/stats_build.go b/pkg/scan/events/stats_build.go new file mode 100644 index 0000000000..0f01724411 --- /dev/null +++ b/pkg/scan/events/stats_build.go @@ -0,0 +1,80 @@ +//go:build stats +// +build stats + +package events + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +var _ ScanEventWorker = &ScanStatsWorker{} + +var defaultWorker = &ScanStatsWorker{} + +// ScanStatsWorker is a worker for scanning stats +// This tracks basic stats in jsonlines format +// in given directory or a default directory with name stats_{timestamp} in the current directory +type ScanStatsWorker struct { + config *ScanConfig + m *sync.Mutex + directory string + enc *json.Encoder +} + +// Init initializes the scan stats worker +func InitWithConfig(config *ScanConfig, statsDirectory string) { + currentTime := time.Now().Format("20060102150405") + dirName := fmt.Sprintf("nuclei-stats-%s", currentTime) + err := os.Mkdir(dirName, 0755) + if err != nil { + panic(err) + } + // save the config to the directory + bin, err := json.MarshalIndent(config, "", " ") + if err != nil { + panic(err) + } + err = os.WriteFile(filepath.Join(dirName, ConfigFile), bin, 0755) + if err != nil { + panic(err) + } + defaultWorker = &ScanStatsWorker{config: config, m: &sync.Mutex{}, directory: dirName} + err = defaultWorker.initEventsFile() + if err != nil { + panic(err) + } +} + +// initEventsFile initializes the events file for the worker +func (s *ScanStatsWorker) initEventsFile() error { + f, err := os.Create(filepath.Join(s.directory, EventsFile)) + if err != nil { + return err + } + s.enc = json.NewEncoder(f) + return nil +} + +// AddScanEvent adds a scan event to the worker +func (s *ScanStatsWorker) AddScanEvent(event ScanEvent) { + s.m.Lock() + defer s.m.Unlock() + + err := s.enc.Encode(event) + if err != nil { + panic(err) + } +} + +// AddScanEvent adds a scan event to the worker +func AddScanEvent(event ScanEvent) { + if defaultWorker == nil { + return + } + defaultWorker.AddScanEvent(event) +} diff --git a/pkg/scan/events/utils.go b/pkg/scan/events/utils.go new file mode 100644 index 0000000000..edd7b09ae5 --- /dev/null +++ b/pkg/scan/events/utils.go @@ -0,0 +1,45 @@ +package events + +import ( + "time" +) + +type ScanEventWorker interface { + // AddScanEvent adds a scan event to the worker + AddScanEvent(event ScanEvent) +} + +// Track scan start / finish status +type ScanStatus string + +const ( + ScanStarted ScanStatus = "scan_start" + ScanFinished ScanStatus = "scan_end" +) + +const ( + ConfigFile = "config.json" + EventsFile = "events.jsonl" +) + +// ScanEvent represents a single scan event with its metadata +type ScanEvent struct { + Target string `json:"target" yaml:"target"` + TemplateType string `json:"template_type" yaml:"template_type"` + TemplateID string `json:"template_id" yaml:"template_id"` + TemplatePath string `json:"template_path" yaml:"template_path"` + MaxRequests int `json:"max_requests" yaml:"max_requests"` + Time time.Time `json:"time" yaml:"time"` + EventType ScanStatus `json:"event_type" yaml:"event_type"` +} + +// ScanConfig is only in context of scan event analysis +type ScanConfig struct { + Name string `json:"name" yaml:"name"` + TargetCount int `json:"target_count" yaml:"target_count"` + TemplatesCount int `json:"templates_count" yaml:"templates_count"` + TemplateConcurrency int `json:"template_concurrency" yaml:"template_concurrency"` + PayloadConcurrency int `json:"payload_concurrency" yaml:"payload_concurrency"` + JsConcurrency int `json:"js_concurrency" yaml:"js_concurrency"` + Retries int `json:"retries" yaml:"retries"` +} diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index a5e310b7c6..8851349019 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" ) + type ScanContextOption func(*ScanContext) func WithEvents() ScanContextOption { diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index e434f2173c..fc504171e6 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "sync/atomic" + "time" "github.com/dop251/goja" "github.com/projectdiscovery/gologger" @@ -14,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v3/pkg/scan" + "github.com/projectdiscovery/nuclei/v3/pkg/scan/events" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" @@ -92,6 +94,31 @@ func (e *TemplateExecuter) Requests() int { // Execute executes the protocol group and returns true or false if results were found. func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { + + // === when nuclei is built with -tags=stats === + // Note: this is no-op (empty functions) when nuclei is built in normal or without -tags=stats + events.AddScanEvent(events.ScanEvent{ + Target: ctx.Input.MetaInput.Input, + Time: time.Now(), + EventType: events.ScanStarted, + TemplateType: e.getTemplateType(), + TemplateID: e.options.TemplateID, + TemplatePath: e.options.TemplatePath, + MaxRequests: e.Requests(), + }) + defer func() { + events.AddScanEvent(events.ScanEvent{ + Target: ctx.Input.MetaInput.Input, + Time: time.Now(), + EventType: events.ScanFinished, + TemplateType: e.getTemplateType(), + TemplateID: e.options.TemplateID, + TemplatePath: e.options.TemplatePath, + MaxRequests: e.Requests(), + }) + }() + // ==== end of stats ==== + // executed contains status of execution if it was successfully executed or not // doesn't matter if it was matched or not executed := &atomic.Bool{} @@ -182,3 +209,17 @@ func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output. ctx.LogError(err) return ctx.GenerateResult(), err } + +// getTemplateType returns the template type of the template +func (e *TemplateExecuter) getTemplateType() string { + if len(e.requests) == 0 { + return "null" + } + if e.options.Flow != "" { + return "flow" + } + if len(e.requests) > 1 { + return "multiprotocol" + } + return e.requests[0].Type().String() +} diff --git a/pkg/types/types.go b/pkg/types/types.go index c251dc0c85..b6bdcaee55 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -419,7 +419,9 @@ func DefaultOptions() *Options { BulkSize: 25, TemplateThreads: 25, HeadlessBulkSize: 10, + PayloadConcurrency: 25, HeadlessTemplateThreads: 10, + ProbeConcurrency: 50, Timeout: 5, Retries: 1, MaxHostError: 30,