Skip to content

Commit

Permalink
nuclei 'stats' build : scan events + chart utils (#5032)
Browse files Browse the repository at this point in the history
* prototype new scan events

* scan-event: improvements + conditional build

* add scan charts server: make scan-charts

* scan-charts: bug fix
  • Loading branch information
tarunKoyalwar authored Apr 16, 2024
1 parent bec7cb2 commit ea2e13a
Show file tree
Hide file tree
Showing 14 changed files with 692 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,8 @@ pkg/protocols/headless/engine/.cache
/fuzzplayground
integration_tests/fuzzplayground
/dsl.md
/nuclei-stats
/nuclei-stats-*
/scan-charts


9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,18 @@ ifneq ($(shell go env GOOS),darwin)
LDFLAGS := -extldflags "-static"
endif

.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs

all: build
build:
rm -f nuclei 2>/dev/null
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
build-stats:
rm -f nuclei-stats 2>/dev/null
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
scan-charts:
rm -f scan-charts 2>/dev/null
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
docs:
if ! which dstdocgen > /dev/null; then
echo -e "Command not found! Install? (y/n) \c"
Expand Down
40 changes: 40 additions & 0 deletions cmd/scan-charts/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package main

import (
"flag"

"github.com/projectdiscovery/nuclei/v3/pkg/scan/charts"
)

var (
dir string
address string
output string
)

func main() {
flag.StringVar(&dir, "dir", "", "directory to scan")
flag.StringVar(&address, "address", ":9000", "address to run the server on")
flag.StringVar(&output, "output", "", "output filename of generated html file")
flag.Parse()

if dir == "" {
flag.Usage()
return
}

server, err := charts.NewScanEventsCharts(dir)
if err != nil {
panic(err)
}
server.PrintInfo()

if output != "" {
if err = server.GenerateHTML(output); err != nil {
panic(err)
}
return
}

server.Start(address)
}
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ require (
github.com/aws/smithy-go v1.13.5 // indirect
github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-echarts/go-echarts/v2 v2.3.3
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
Expand All @@ -348,3 +349,5 @@ require (

// https://go.dev/ref/mod#go-mod-file-retract
retract v3.2.0 // retract due to broken js protocol issue

replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg=
github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
Expand Down Expand Up @@ -1013,6 +1015,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
Expand Down
15 changes: 15 additions & 0 deletions internal/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/installer"
"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
uncoverlib "github.com/projectdiscovery/uncover"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/env"
Expand Down Expand Up @@ -553,6 +554,20 @@ func (r *Runner) RunEnumeration() error {
executorOpts.InputHelper.InputsHTTP = inputHelpers
}

// initialize stats worker ( this is no-op unless nuclei is built with stats build tag)
// during execution a directory with 2 files will be created in the current directory
// config.json - containing below info
// events.jsonl - containing all start and end times of all templates
events.InitWithConfig(&events.ScanConfig{
Name: "nuclei-stats", // make this configurable
TargetCount: int(r.inputProvider.Count()),
TemplatesCount: len(store.Templates()) + len(store.Workflows()),
TemplateConcurrency: r.options.TemplateThreads,
PayloadConcurrency: r.options.PayloadConcurrency,
JsConcurrency: r.options.JsConcurrency,
Retries: r.options.Retries,
}, "")

enumeration := false
var results *atomic.Bool
results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)
Expand Down
87 changes: 87 additions & 0 deletions pkg/scan/charts/charts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package charts

import (
"encoding/json"
"fmt"
"os"
"path/filepath"

"github.com/labstack/echo/v4"
"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
fileutil "github.com/projectdiscovery/utils/file"
)

// ScanEventsCharts is a struct for nuclei event charts
type ScanEventsCharts struct {
eventsDir string
config *events.ScanConfig
data []events.ScanEvent
}

func (sc *ScanEventsCharts) PrintInfo() {
fmt.Printf("[+] Scan Info\n")
fmt.Printf(" - Name: %s\n", sc.config.Name)
fmt.Printf(" - Target Count: %d\n", sc.config.TargetCount)
fmt.Printf(" - Template Count: %d\n", sc.config.TemplatesCount)
fmt.Printf(" - Template Concurrency: %d\n", sc.config.TemplateConcurrency)
fmt.Printf(" - Payload Concurrency: %d\n", sc.config.PayloadConcurrency)
fmt.Printf(" - Retries: %v\n", sc.config.Retries)
fmt.Printf(" - Total Events: %d\n", len(sc.data))
fmt.Println()
}

// NewScanEventsCharts creates a new nuclei event charts
func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) {
sc := &ScanEventsCharts{eventsDir: eventsDir}
if !fileutil.FolderExists(eventsDir) {
return nil, fmt.Errorf("events directory does not exist")
}
// open two files
// config.json
bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile))
if err != nil {
return nil, err
}
var config events.ScanConfig
err = json.Unmarshal(bin, &config)
if err != nil {
return nil, err
}
sc.config = &config

// events.jsonl
f, err := os.Open(filepath.Join(eventsDir, events.EventsFile))
if err != nil {
return nil, err
}
defer f.Close()

data := []events.ScanEvent{}
dec := json.NewDecoder(f)
for {
var event events.ScanEvent
if err := dec.Decode(&event); err != nil {
break
}
data = append(data, event)
}
sc.data = data

if len(data) == 0 {
return nil, fmt.Errorf("no events found in the events file")
}

return sc, nil
}

// Start starts the nuclei event charts server
func (sc *ScanEventsCharts) Start(addr string) {
e := echo.New()
e.HideBanner = true
e.GET("/concurrency", sc.ConcurrencyVsTime)
e.GET("/requests", sc.TotalRequestsOverTime)
e.GET("/slow", sc.TopSlowTemplates)
e.GET("/rps", sc.RequestsVSInterval)
e.GET("/", sc.AllCharts)
e.Logger.Fatal(e.Start(addr))
}
Loading

0 comments on commit ea2e13a

Please sign in to comment.