From 694835c459517250c661399e0320e74f1db3387e Mon Sep 17 00:00:00 2001 From: Douglas Danger Manley <127235272+doug-threatmate@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:11:22 -0400 Subject: [PATCH] Add more support for `fs.FS` in template parsing (#5421) * misc update * chore(deps): bump github.com/gin-gonic/gin from 1.9.0 to 1.9.1 (#4252) Bumps [github.com/gin-gonic/gin](https://github.com/gin-gonic/gin) from 1.9.0 to 1.9.1. - [Release notes](https://github.com/gin-gonic/gin/releases) - [Changelog](https://github.com/gin-gonic/gin/blob/master/CHANGELOG.md) - [Commits](https://github.com/gin-gonic/gin/compare/v1.9.0...v1.9.1) --- updated-dependencies: - dependency-name: github.com/gin-gonic/gin dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/docker/docker (#4316) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.5+incompatible to 24.0.7+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v24.0.5...v24.0.7) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix README_CN.md typos (#4369) * version update * Add more support for `fs.FS` in the disk catalog This adds more support for `fs.FS` in the disk catalog. This fixes some places where direct `os` file-related calls were being made to use the catalog interface instead. Note that the JavaScript compiler *still* does not work in any context where the `pkg/js/libs/fs` package is used. In particular, the `ReadFilesFromDir` function is hard-coded to use the `os` package and not respect the catalog. * Remove some testing artifacts * Wrap up * Unwind other changes * Add a LoadHelperFileFunction to Options * Use a direct func * Tweak validation * Use a function type --------- Signed-off-by: dependabot[bot] Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Xc1Ym Co-authored-by: Sandeep Singh --- pkg/catalog/disk/find.go | 13 +++++++- pkg/catalog/disk/path.go | 34 +++++++++++++++------ pkg/protocols/common/generators/validate.go | 7 +++++ pkg/types/types.go | 21 +++++++++++-- 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/pkg/catalog/disk/find.go b/pkg/catalog/disk/find.go index a5d46ff078..0e4021b87c 100644 --- a/pkg/catalog/disk/find.go +++ b/pkg/catalog/disk/find.go @@ -216,12 +216,17 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s // is a file, it returns true otherwise false with no errors. func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) { if c.templatesFS != nil { - absPath = strings.TrimPrefix(absPath, "/") + absPath = strings.Trim(absPath, "/") } var info fs.File if c.templatesFS == nil { info, err = os.Open(absPath) } else { + // If we were given no path, then it's not a file, it's the root, and we can quietly return. + if absPath == "" { + return "", false, nil + } + info, err = c.templatesFS.Open(absPath) } if err != nil { @@ -263,6 +268,12 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string] }, ) } else { + // For the special case of the root directory, we need to pass "." to `fs.WalkDir`. + if absPath == "" { + absPath = "." + } + absPath = strings.TrimSuffix(absPath, "/") + err = fs.WalkDir( c.templatesFS, absPath, diff --git a/pkg/catalog/disk/path.go b/pkg/catalog/disk/path.go index a55faee8f6..5c4b6dbb95 100644 --- a/pkg/catalog/disk/path.go +++ b/pkg/catalog/disk/path.go @@ -2,6 +2,7 @@ package disk import ( "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -21,6 +22,11 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) { if filepath.IsAbs(templateName) { return templateName, nil } + if c.templatesFS != nil { + if potentialPath, err := c.tryResolve(templateName); err != errNoValidCombination { + return potentialPath, nil + } + } if second != "" { secondBasePath := filepath.Join(filepath.Dir(second), templateName) if potentialPath, err := c.tryResolve(secondBasePath); err != errNoValidCombination { @@ -28,17 +34,19 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) { } } - curDirectory, err := os.Getwd() - if err != nil { - return "", err - } + if c.templatesFS == nil { + curDirectory, err := os.Getwd() + if err != nil { + return "", err + } - templatePath := filepath.Join(curDirectory, templateName) - if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { - return potentialPath, nil + templatePath := filepath.Join(curDirectory, templateName) + if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { + return potentialPath, nil + } } - templatePath = filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName) + templatePath := filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName) if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination { return potentialPath, nil } @@ -50,8 +58,14 @@ var errNoValidCombination = errors.New("no valid combination found") // tryResolve attempts to load locate the target by iterating across all the folders tree func (c *DiskCatalog) tryResolve(fullPath string) (string, error) { - if fileutil.FileOrFolderExists(fullPath) { - return fullPath, nil + if c.templatesFS == nil { + if fileutil.FileOrFolderExists(fullPath) { + return fullPath, nil + } + } else { + if _, err := fs.Stat(c.templatesFS, fullPath); err == nil { + return fullPath, nil + } } return "", errNoValidCombination } diff --git a/pkg/protocols/common/generators/validate.go b/pkg/protocols/common/generators/validate.go index adb537d289..0aa0737149 100644 --- a/pkg/protocols/common/generators/validate.go +++ b/pkg/protocols/common/generators/validate.go @@ -22,6 +22,13 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat return errors.New("invalid number of lines in payload") } + // For historical reasons, "validate" checks to see if the payload file exist. + // If we're using a custom helper function, then we need to skip any validation beyond just checking the string syntax. + // Actually attempting to load the file will determine whether or not it exists. + if g.options.LoadHelperFileFunction != nil { + return nil + } + // check if it's a file and try to load it if fileutil.FileExists(payloadType) { continue diff --git a/pkg/types/types.go b/pkg/types/types.go index 51375e7342..9cc88f49ff 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -23,6 +23,9 @@ var ( ErrNoMoreRequests = io.EOF ) +// LoadHelperFileFunction can be used to load a helper file. +type LoadHelperFileFunction func(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) + // Options contains the configuration options for nuclei scanner. type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths @@ -408,6 +411,9 @@ type Options struct { HttpApiEndpoint string // ListTemplateProfiles lists all available template profiles ListTemplateProfiles bool + // LoadHelperFileFunction is a function that will be used to execute LoadHelperFile. + // If none is provided, then the default implementation will be used. + LoadHelperFileFunction LoadHelperFileFunction // timeouts contains various types of timeouts used in nuclei // these timeouts are derived from dial-timeout (-timeout) with known multipliers // This is internally managed and does not need to be set by user by explicitly setting @@ -540,10 +546,21 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { return optionalArguments } -// LoadHelperFile loads a helper file needed for the template +// LoadHelperFile loads a helper file needed for the template. +// +// If LoadHelperFileFunction is set, then that function will be used. +// Otherwise, the default implementation will be used, which respects the sandbox rules and only loads files from allowed directories. +func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { + if options.LoadHelperFileFunction != nil { + return options.LoadHelperFileFunction(helperFile, templatePath, catalog) + } + return options.defaultLoadHelperFile(helperFile, templatePath, catalog) +} + +// defaultLoadHelperFile loads a helper file needed for the template // this respects the sandbox rules and only loads files from // allowed directories -func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { +func (options *Options) defaultLoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { if !options.AllowLocalFileAccess { // if global file access is disabled try loading with restrictions absPath, err := options.GetValidAbsPath(helperFile, templatePath)