From deeaae366d42e1e882cd10927ccb2401948736b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 4 Jul 2024 22:51:39 +0300 Subject: [PATCH 1/5] disable non default templates as default --- go.mod | 1 + go.sum | 2 ++ internal/runner/runner.go | 12 ++++++++++++ pkg/catalog/loader/loader.go | 17 +++++++++++++++-- pkg/templates/parser_stats.go | 2 ++ pkg/templates/stats.go | 2 ++ pkg/templates/templates.go | 5 +++++ 7 files changed, 39 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4dba575ed8..5ae0d9bd0b 100644 --- a/go.mod +++ b/go.mod @@ -206,6 +206,7 @@ require ( github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/refraction-networking/utls v1.6.6 // indirect + github.com/samber/lo v1.44.0 // indirect github.com/sashabaranov/go-openai v1.15.3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/go.sum b/go.sum index 8a5ed5c0de..eb8a310193 100644 --- a/go.sum +++ b/go.sum @@ -948,6 +948,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samber/lo v1.44.0 h1:5il56KxRE+GHsm1IR+sZ/6J42NODigFiqCWpSc2dybA= +github.com/samber/lo v1.44.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sashabaranov/go-openai v1.15.3 h1:rzoNK9n+Cak+PM6OQ9puxDmFllxfnVea9StlmhglXqA= github.com/sashabaranov/go-openai v1.15.3/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index bc436500af..74edc90221 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -26,6 +26,7 @@ import ( fileutil "github.com/projectdiscovery/utils/file" permissionutil "github.com/projectdiscovery/utils/permission" updateutils "github.com/projectdiscovery/utils/update" + "github.com/samber/lo" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -519,6 +520,15 @@ func (r *Runner) RunEnumeration() error { // if input type is not list (implicitly enable fuzzing) r.options.DAST = true } + + if lo.NoneBy(r.options.Targets, func(target string) bool { return fileutil.FileOrFolderExists(target) }) { + loaderConfig.ExcludeFileProtocol = true + } + + if r.options.Vars.IsEmpty() && !r.options.EnvironmentVariables { + loaderConfig.ExcludeSelfContained = true + } + store, err := loader.New(loaderConfig) if err != nil { return errors.Wrap(err, "Could not create loader.") @@ -727,6 +737,8 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.DisplayAsWarning(httpProtocol.SetThreadToCountZero) stats.ForceDisplayWarning(templates.SkippedUnsignedStats) stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats) + stats.ForceDisplayWarning(templates.SkippedSelfContainedStats) + stats.ForceDisplayWarning(templates.SkippedFileStats) cfg := config.DefaultConfig diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index e1d8371e35..7374dd3360 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -60,8 +60,10 @@ type Config struct { ExcludeIds []string IncludeConditions []string - Catalog catalog.Catalog - ExecutorOptions protocols.ExecutorOptions + Catalog catalog.Catalog + ExecutorOptions protocols.ExecutorOptions + ExcludeSelfContained bool + ExcludeFileProtocol bool } // Store is a storage for loaded nuclei templates @@ -465,6 +467,17 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ stats.Increment(templates.SkippedUnsignedStats) return } + + if parsed.SelfContained && store.config.ExcludeSelfContained { + stats.Increment(templates.SkippedSelfContainedStats) + return + } + + if parsed.HasFileProtocol() && store.config.ExcludeFileProtocol { + stats.Increment(templates.SkippedFileStats) + return + } + // if template has request signature like aws then only signed and verified templates are allowed if parsed.UsesRequestSignature() && !parsed.Verified { stats.Increment(templates.SkippedRequestSignatureStats) diff --git a/pkg/templates/parser_stats.go b/pkg/templates/parser_stats.go index 290601032d..4fd015a54b 100644 --- a/pkg/templates/parser_stats.go +++ b/pkg/templates/parser_stats.go @@ -11,4 +11,6 @@ const ( ExludedDastTmplStats = "fuzz-flag-missing-warnings" SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates SkippedRequestSignatureStats = "skipped-request-signature-stats" + SkippedSelfContainedStats = "skipped-self-contained-stats" + SkippedFileStats = "skipped-file-stats" ) diff --git a/pkg/templates/stats.go b/pkg/templates/stats.go index fe3d4ca339..a54d7a9376 100644 --- a/pkg/templates/stats.go +++ b/pkg/templates/stats.go @@ -13,4 +13,6 @@ func init() { stats.NewEntry(ExludedDastTmplStats, "Excluded %d dast template[s] (disabled as default), use -dast option to run dast templates.") stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]") stats.NewEntry(SkippedRequestSignatureStats, "Skipping %d templates, HTTP Request signatures can only be used in Signed & Verified templates.") + stats.NewEntry(SkippedSelfContainedStats, "Skipping %d self-contained template[s], use -var or -env-vars flag to run them") + stats.NewEntry(SkippedFileStats, "Skipping %d file template[s], use file or directory as an input to run file templates") } diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index df0c47648d..0869d3f79f 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -227,6 +227,11 @@ func (template *Template) HasCodeProtocol() bool { return len(template.RequestsCode) > 0 } +// HasFileProtocol returns true if the template has a file protocol section +func (template *Template) HasFileProtocol() bool { + return len(template.RequestsFile) > 0 +} + // validateAllRequestIDs check if that protocol already has given id if not // then is is manually set to proto_index func (template *Template) validateAllRequestIDs() { From 5477dc0690190b72bc784087c1a926deaa30c25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 4 Jul 2024 23:32:42 +0300 Subject: [PATCH 2/5] fix failing tests --- cmd/integration-test/code.go | 2 +- cmd/integration-test/http.go | 2 +- cmd/integration-test/network.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/integration-test/code.go b/cmd/integration-test/code.go index 94c28ea001..a72b3968f3 100644 --- a/cmd/integration-test/code.go +++ b/cmd/integration-test/code.go @@ -99,7 +99,7 @@ type codePreCondition struct{} // Execute executes a test case and returns an error if occurred func (h *codePreCondition) Execute(filePath string) error { - results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code") + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code", "-env-var") if err != nil { return err } diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 5b218aa53b..9efff41650 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -952,7 +952,7 @@ func (h *httpRequestSelfContained) Execute(filePath string) error { }() defer server.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-var foo=bar") if err != nil { return err } diff --git a/cmd/integration-test/network.go b/cmd/integration-test/network.go index 8081c973e9..14266512c6 100644 --- a/cmd/integration-test/network.go +++ b/cmd/integration-test/network.go @@ -119,7 +119,7 @@ func (h *networkRequestSelContained) Execute(filePath string) error { _, _ = conn.Write([]byte("Authentication successful")) }) defer ts.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-var foo=bar") if err != nil { return err } From 1ef1a071851e4aebb6fa4a4ae38b3c3a9e0d3342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 4 Jul 2024 23:49:07 +0300 Subject: [PATCH 3/5] minor --- cmd/integration-test/code.go | 2 +- cmd/integration-test/http.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/integration-test/code.go b/cmd/integration-test/code.go index a72b3968f3..45bc42c937 100644 --- a/cmd/integration-test/code.go +++ b/cmd/integration-test/code.go @@ -99,7 +99,7 @@ type codePreCondition struct{} // Execute executes a test case and returns an error if occurred func (h *codePreCondition) Execute(filePath string) error { - results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code", "-env-var") + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code", "-var", "foo=bar") if err != nil { return err } diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 9efff41650..d7b1583378 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -988,7 +988,7 @@ func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error { }() defer server.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-var foo=bar") if err != nil { return err } From c387b4c2d087da2a65aa36c034e9f207464b62c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 5 Jul 2024 14:03:03 +0300 Subject: [PATCH 4/5] filter out templates contains vars --- pkg/catalog/loader/loader.go | 54 +++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 7374dd3360..a5ab24f12d 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -5,10 +5,12 @@ import ( "io" "net/url" "os" + "regexp" "sort" "strings" "sync" + "github.com/Knetic/govaluate" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -16,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -468,7 +471,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ return } - if parsed.SelfContained && store.config.ExcludeSelfContained { + if parsed.SelfContained && store.config.ExcludeSelfContained && templateContainsUnresolvedVariables(templatePath) { stats.Increment(templates.SkippedSelfContainedStats) return } @@ -541,6 +544,55 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ return loadedTemplates.Slice } +var ( + numericalExpressionRegex = regexp.MustCompile(`^[0-9+\-/\W]+$`) + unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`) +) + +// copy of the original function from pkg/protocols/common/expressions/variables.go:ContainsUnresolvedVariables +func templateContainsUnresolvedVariables(templatePath string) bool { + data, err := os.ReadFile(templatePath) + if err != nil { + return false + } + + matches := unresolvedVariablesRegex.FindAllStringSubmatch(string(data), -1) + if len(matches) == 0 { + return false + } + + var unresolvedVariables []string + for _, match := range matches { + if len(match) < 2 { + continue + } + + // Skip if the match is an expression + if numericalExpressionRegex.MatchString(match[1]) { + continue + } + // or if it contains only literals (can be solved from expression engine) + if hasLiteralsOnly(match[1]) { + continue + } + unresolvedVariables = append(unresolvedVariables, match[1]) + } + + return len(unresolvedVariables) > 0 +} + +func hasLiteralsOnly(data string) bool { + expr, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions) + if err != nil { + return false + } + if expr != nil { + _, err = expr.Evaluate(nil) + return err == nil + } + return true +} + // IsHTTPBasedProtocolUsed returns true if http/headless protocol is being used for // any templates. func IsHTTPBasedProtocolUsed(store *Store) bool { From 5dd0d351d9043a52ec5224be3d0a738ffdc5e737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 5 Jul 2024 14:03:42 +0300 Subject: [PATCH 5/5] move things around --- internal/runner/runner.go | 9 --------- pkg/catalog/loader/loader.go | 17 +++++++++++------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 74edc90221..74be3dc6ef 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -26,7 +26,6 @@ import ( fileutil "github.com/projectdiscovery/utils/file" permissionutil "github.com/projectdiscovery/utils/permission" updateutils "github.com/projectdiscovery/utils/update" - "github.com/samber/lo" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -521,14 +520,6 @@ func (r *Runner) RunEnumeration() error { r.options.DAST = true } - if lo.NoneBy(r.options.Targets, func(target string) bool { return fileutil.FileOrFolderExists(target) }) { - loaderConfig.ExcludeFileProtocol = true - } - - if r.options.Vars.IsEmpty() && !r.options.EnvironmentVariables { - loaderConfig.ExcludeSelfContained = true - } - store, err := loader.New(loaderConfig) if err != nil { return errors.Wrap(err, "Could not create loader.") diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index a5ab24f12d..0fd0d34f1f 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -27,9 +27,11 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/workflows" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" + "github.com/samber/lo" ) const ( @@ -63,10 +65,8 @@ type Config struct { ExcludeIds []string IncludeConditions []string - Catalog catalog.Catalog - ExecutorOptions protocols.ExecutorOptions - ExcludeSelfContained bool - ExcludeFileProtocol bool + Catalog catalog.Catalog + ExecutorOptions protocols.ExecutorOptions } // Store is a storage for loaded nuclei templates @@ -471,12 +471,17 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ return } - if parsed.SelfContained && store.config.ExcludeSelfContained && templateContainsUnresolvedVariables(templatePath) { + if parsed.SelfContained && + store.config.ExecutorOptions.Options.Vars.IsEmpty() && !store.config.ExecutorOptions.Options.EnvironmentVariables && + templateContainsUnresolvedVariables(templatePath) { stats.Increment(templates.SkippedSelfContainedStats) return } - if parsed.HasFileProtocol() && store.config.ExcludeFileProtocol { + if parsed.HasFileProtocol() && + lo.NoneBy(store.config.ExecutorOptions.Options.Targets, func(target string) bool { + return fileutil.FileOrFolderExists(target) + }) { stats.Increment(templates.SkippedFileStats) return }