SDK: abstracted and minimal nuclei v3 sdk (#4104)

* new sdk progress

* nuclei v3 new sdk/library

* fix TestActionGetResource broken link

* fix clistats + clustering and more

* fix lint error

* fix missing ticker

* update advanced library usage example

* fix integration tests

* misc update

* add utm_source and fix lint error

---------

Co-authored-by: sandeep <[email protected]>

.github/workflows/build-test.yml

@@ -56,6 +56,10 @@ jobs:
         run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
         working-directory: v2/cmd/nuclei/
 
-      - name: Example Code Tests
-        run: go build .
-        working-directory: v2/examples/
+      - name: Example SDK Simple
+        run: go run .
+        working-directory: v2/examples/simple/
+
+      - name: Example SDK Advanced
+        run: go run .
+        working-directory: v2/examples/advanced/

README.md

@@ -379,7 +379,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
 
 ### Using Nuclei From Go Code
 
-Examples of using Nuclei From Go Code to run templates on targets are provided in the [examples](v2/examples/) folder.
+Complete guide of using Nuclei as Library/SDK is available at [lib](v2/lib/README.md)
 
 
 ### Resources

v2/cmd/integration-test/template-path.go

@@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{
 	//template folder path issue
 	{Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}},
 	//cwd
-	{Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}},
+	{Path: "./dns/detect-dangling-cname.yaml", TestCase: &cwdTemplateTest{}},
 	//relative path
 	{Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}},
 	//absolute path

v2/cmd/nuclei/main.go

@@ -361,7 +361,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
 		flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
-		flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"),
 		flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"),
 	)
 

v2/examples/advanced/advanced.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+func main() {
+	// create nuclei engine with options
+	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	if err != nil {
+		panic(err)
+	}
+	// setup sizedWaitgroup to handle concurrency
+	sg := sizedwaitgroup.New(10)
+
+	// scan 1 = run dns templates on scanme.sh
+	sg.Add()
+	go func() {
+		defer sg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"},
+			nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
+		)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// scan 2 = run templates with oast tags on honey.scanme.sh
+	sg.Add()
+	go func() {
+		defer sg.Done()
+		err = ne.ExecuteNucleiWithOpts([]string{"http://honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}))
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// wait for all scans to finish
+	sg.Wait()
+	defer ne.Close()
+
+	// Output:
+	// [dns-saas-service-detection] scanme.sh
+	// [nameserver-fingerprint] scanme.sh
+	// [dns-saas-service-detection] honey.scanme.sh
+}

v2/examples/simple/simple.go

@@ -0,0 +1,20 @@
+package main
+
+import nuclei "github.com/projectdiscovery/nuclei/v2/lib"
+
+func main() {
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
+		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), // optionally enable metrics server for better observability
+	)
+	if err != nil {
+		panic(err)
+	}
+	// load targets and optionally probe non http/https targets
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+}

v2/internal/installer/util.go

@@ -103,3 +103,29 @@ func isEmptyDir(dir string) bool {
 	})
 	return !hasFiles
 }
+
+// getUtmSource returns utm_source from environment variable
+func getUtmSource() string {
+	value := ""
+	switch {
+	case os.Getenv("GH_ACTION") != "":
+		value = "ghci"
+	case os.Getenv("TRAVIS") != "":
+		value = "travis"
+	case os.Getenv("CIRCLECI") != "":
+		value = "circleci"
+	case os.Getenv("CI") != "":
+		value = "gitlabci" // this also includes bitbucket
+	case os.Getenv("GITHUB_ACTIONS") != "":
+		value = "ghci"
+	case os.Getenv("AWS_EXECUTION_ENV") != "":
+		value = os.Getenv("AWS_EXECUTION_ENV")
+	case os.Getenv("JENKINS_URL") != "":
+		value = "jenkins"
+	case os.Getenv("FUNCTION_TARGET") != "":
+		value = "gcf"
+	default:
+		value = "unknown"
+	}
+	return value
+}

v2/internal/installer/versioncheck.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"sync"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -34,7 +35,56 @@ type PdtmAPIResponse struct {
 // and returns an error if it fails to check on success it returns nil and changes are
 // made to the default config in config.DefaultConfig
 func NucleiVersionCheck() error {
-	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams())
+	return doVersionCheck(false)
+}
+
+// this will be updated by features of 1.21 release (which directly provides sync.Once(func()))
+type sdkUpdateCheck struct {
+	sync.Once
+}
+
+var sdkUpdateCheckInstance = &sdkUpdateCheck{}
+
+// NucleiSDKVersionCheck checks for latest version of nuclei which running in sdk mode
+// this only happens once per process regardless of how many times this function is called
+func NucleiSDKVersionCheck() {
+	sdkUpdateCheckInstance.Do(func() {
+		_ = doVersionCheck(true)
+	})
+}
+
+// getpdtmParams returns encoded query parameters sent to update check endpoint
+func getpdtmParams(isSDK bool) string {
+	params := &url.Values{}
+	params.Add("os", runtime.GOOS)
+	params.Add("arch", runtime.GOARCH)
+	params.Add("go_version", runtime.Version())
+	params.Add("v", config.Version)
+	if isSDK {
+		params.Add("sdk", "true")
+	}
+	params.Add("utm_source", getUtmSource())
+	return params.Encode()
+}
+
+// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
+func UpdateIgnoreFile() error {
+	resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams(false))
+	if err != nil {
+		return err
+	}
+	bin, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil {
+		return err
+	}
+	return config.DefaultConfig.UpdateNucleiIgnoreHash()
+}
+
+func doVersionCheck(isSDK bool) error {
+	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
 	if err != nil {
 		return err
 	}
@@ -63,29 +113,3 @@ func NucleiVersionCheck() error {
 	}
 	return config.DefaultConfig.WriteVersionCheckData(pdtmResp.IgnoreHash, nucleiversion, templateversion)
 }
-
-// getpdtmParams returns encoded query parameters sent to update check endpoint
-func getpdtmParams() string {
-	params := &url.Values{}
-	params.Add("os", runtime.GOOS)
-	params.Add("arch", runtime.GOARCH)
-	params.Add("go_version", runtime.Version())
-	params.Add("v", config.Version)
-	return params.Encode()
-}
-
-// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
-func UpdateIgnoreFile() error {
-	resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams())
-	if err != nil {
-		return err
-	}
-	bin, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil {
-		return err
-	}
-	return config.DefaultConfig.UpdateNucleiIgnoreHash()
-}

v2/internal/runner/options.go

@@ -71,7 +71,7 @@ func ParseOptions(options *types.Options) {
 	}
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
-	if err := validateOptions(options); err != nil {
+	if err := ValidateOptions(options); err != nil {
 		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
 
@@ -105,7 +105,7 @@ func ParseOptions(options *types.Options) {
 }
 
 // validateOptions validates the configuration options passed
-func validateOptions(options *types.Options) error {
+func ValidateOptions(options *types.Options) error {
 	validate := validator.New()
 	if err := validate.Struct(options); err != nil {
 		if _, ok := err.(*validator.InvalidValidationError); ok {