Skip to content

Commit

Permalink
Environment variable controlled template downloads
Browse files Browse the repository at this point in the history
  • Loading branch information
@kchason
kchason committed Jul 11, 2023
1 parent f35a6fe commit c672ee8
Show file tree
Hide file tree
Showing 5 changed files with 84 additions and 68 deletions.
123 changes: 67 additions & 56 deletions v2/internal/installer/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ func (t *templateUpdateResults) String() string {
// TemplateManager is a manager for templates.
// It downloads / updates / installs templates.
type TemplateManager struct {
CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates
CustomTemplates *customtemplates.CustomTemplatesManager // optional if given tries to download custom templates
DisablePublicTemplates bool // if true,
// public templates are not downloaded from the GitHub nuclei-templates repository
}

// FreshInstallIfNotExists installs templates if they are not already installed
Expand Down Expand Up @@ -95,59 +97,68 @@ func (t *TemplateManager) installTemplatesAt(dir string) error {
return errorutil.NewWithErr(err).Msgf("failed to create directory at %s", dir)
}
}
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
}
// write templates to disk
if err := t.writeTemplatestoDisk(ghrd, dir); err != nil {
return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir)
if !t.DisablePublicTemplates {
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
}
// write templates to disk
if err := t.writeTemplatesToDisk(ghrd, dir); err != nil {
return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir)
}
gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir)
} else {
gologger.Info().Msgf("Skipping installation of public nuclei-templates")
}
gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir)
return nil
}

// updateTemplatesAt updates templates at given directory
func (t *TemplateManager) updateTemplatesAt(dir string) error {
// firstly read checksums from .checksum file these are used to generate stats
oldchecksums, err := t.getChecksumFromDir(dir)
if err != nil {
// if something went wrong overwrite all files
oldchecksums = make(map[string]string)
}
if !t.DisablePublicTemplates {

ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
}
// firstly, read checksums from .checksum file these are used to generate stats
oldchecksums, err := t.getChecksumFromDir(dir)
if err != nil {
// if something went wrong, overwrite all files
oldchecksums = make(map[string]string)
}

gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName())
ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTemplatesRepoName)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir)
}

// write templates to disk
if err := t.writeTemplatestoDisk(ghrd, dir); err != nil {
return err
}
gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName())

// get checksums from new templates
newchecksums, err := t.getChecksumFromDir(dir)
if err != nil {
// unlikely this case will happen
return errorutil.NewWithErr(err).Msgf("failed to get checksums from %s after update", dir)
}
// write templates to disk
if err := t.writeTemplatesToDisk(ghrd, dir); err != nil {
return err
}

// get checksums from new templates
newchecksums, err := t.getChecksumFromDir(dir)
if err != nil {
// unlikely this case will happen
return errorutil.NewWithErr(err).Msgf("failed to get checksums from %s after update", dir)
}

// summarize all changes
results := t.summarizeChanges(oldchecksums, newchecksums)
// summarize all changes
results := t.summarizeChanges(oldchecksums, newchecksums)

// print summary
if results.totalCount > 0 {
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
if !HideUpdateChangesTable {
// print summary table
gologger.Print().Msgf("\nNuclei Templates %s Changelog\n", ghrd.Latest.GetTagName())
gologger.DefaultLogger.Print().Msg(results.String())
// print summary
if results.totalCount > 0 {
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
if !HideUpdateChangesTable {
// print summary table
gologger.Print().Msgf("\nNuclei Templates %s Changelog\n", ghrd.Latest.GetTagName())
gologger.DefaultLogger.Print().Msg(results.String())
}
} else {
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
}
} else {
gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir)
gologger.Info().Msgf("Skipping update of public nuclei-templates")
}
return nil
}
Expand All @@ -173,9 +184,9 @@ func (t *TemplateManager) summarizeChanges(old, new map[string]string) *template
return results
}

// getAbsoluteFilePath returns absolute path where a file should be written based on given uri(i.e files in zip)
// if returned path is empty, it means that file should not be written and skipped
func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.FileInfo) string {
// getAbsoluteFilePath returns an absolute path where a file should be written based on given uri(i.e., files in zip)
// if a returned path is empty, it means that file should not be written and skipped
func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.FileInfo) string {
// overwrite .nuclei-ignore everytime nuclei-templates are downloaded
if f.Name() == config.NucleiIgnoreFileName {
return config.DefaultConfig.GetIgnoreFilePath()
Expand All @@ -194,7 +205,7 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
if index == -1 {
// zip files does not have directory at all , in this case log error but continue
gologger.Warning().Msgf("failed to get directory name from uri: %s", uri)
return filepath.Join(templatedir, uri)
return filepath.Join(templateDir, uri)
}
// seperator is also included in rootDir
rootDirectory := uri[:index+1]
Expand All @@ -205,14 +216,14 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
return ""
}

newPath := filepath.Clean(filepath.Join(templatedir, relPath))
newPath := filepath.Clean(filepath.Join(templateDir, relPath))

if !strings.HasPrefix(newPath, templatedir) {
if !strings.HasPrefix(newPath, templateDir) {
// we don't allow LFI
return ""
}

if newPath == templatedir || newPath == templatedir+string(os.PathSeparator) {
if newPath == templateDir || newPath == templateDir+string(os.PathSeparator) {
// skip writing the folder itself since it already exists
return ""
}
Expand All @@ -228,12 +239,12 @@ func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.File
}

// writeChecksumFileInDir is actual method responsible for writing all templates to directory
func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error {
LocaltemplatesIndex, err := config.GetNucleiTemplatesIndex()
func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error {
localTemplatesIndex, err := config.GetNucleiTemplatesIndex()
if err != nil {
gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err)
if LocaltemplatesIndex == nil {
LocaltemplatesIndex = map[string]string{} // no-op
if localTemplatesIndex == nil {
localTemplatesIndex = map[string]string{} // no-op
}
}

Expand All @@ -253,10 +264,10 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo
// instead of creating it from scratch
id, _ := config.GetTemplateIDFromReader(bytes.NewReader(bin), uri)
if id != "" {
// based on template id, check if we are updating path of official nuclei template
if oldPath, ok := LocaltemplatesIndex[id]; ok {
// based on template id, check if we are updating a path of official nuclei template
if oldPath, ok := localTemplatesIndex[id]; ok {
if oldPath != writePath {
// write new template at new path and delete old template
// write new template at a new path and delete old template
if err := os.WriteFile(writePath, bin, f.Mode()); err != nil {
return errorutil.NewWithErr(err).Msgf("failed to write file %s", uri)
}
Expand Down Expand Up @@ -303,12 +314,12 @@ func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownlo
return errorutil.NewWithErr(err).Msgf("failed to write nuclei templates index")
}

// after installation create and write checksums to .checksum file
// after installation, create and write checksums to .checksum file
return t.writeChecksumFileInDir(dir)
}

// getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension)
// if .checksum file does not exist checksums are calculated and returned
// if .checksum file does not exist, checksums are calculated and returned
func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, error) {
checksumFilePath := config.DefaultConfig.GetChecksumFilePath()
if fileutil.FileExists(checksumFilePath) {
Expand Down
20 changes: 11 additions & 9 deletions v2/internal/runner/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (

func ConfigureOptions() error {
// with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions
// if file has extension `.yaml,.json` we consider those as strings and not files to be read
// if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read
isFromFileFunc := func(s string) bool {
return !config.IsTemplate(s)
}
Expand Down Expand Up @@ -292,7 +292,7 @@ func configureOutput(options *types.Options) {
logutil.DisableDefaultLogger()
}

// loadResolvers loads resolvers from both user provided flag and file
// loadResolvers loads resolvers from both user-provided flags and file
func loadResolvers(options *types.Options) {
if options.ResolversFile == "" {
return
Expand Down Expand Up @@ -398,11 +398,13 @@ func readEnvInputVars(options *types.Options) {
options.AzureServiceURL = os.Getenv("AZURE_SERVICE_URL")

// General options to disable the template download locations from being used.
// This will override the default behavior of downloading templates from the default
// locations as well as the custom locations
options.PublicTemplateDisableDownload = os.Getenv("NUCLEI_TEMPLATES_PUBLIC_DISABLE_DOWNLOAD") == "true"
options.GitHubTemplateDisableDownload = os.Getenv("NUCLEI_TEMPLATES_GITHUB_DISABLE_DOWNLOAD") == "true"
options.GitLabTemplateDisableDownload = os.Getenv("NUCLEI_TEMPLATES_GITLAB_DISABLE_DOWNLOAD") == "true"
options.AwsTemplateDisableDownload = os.Getenv("NUCLEI_TEMPLATES_AWS_DISABLE_DOWNLOAD") == "true"
options.AzureTemplateDisableDownload = os.Getenv("NUCLEI_TEMPLATES_AZURE_DISABLE_DOWNLOAD") == "true"
// This will override the default behavior of downloading templates from the default locations as well as the
// custom locations.
// The primary use-case is when the user wants to use custom templates only and does not want to download any
// templates from the default locations or is unable to connect to the public internet.
options.PublicTemplateDisableDownload = strings.ToLower(os.Getenv("NUCLEI_TEMPLATES_PUBLIC_DISABLE_DOWNLOAD")) == "true"
options.GitHubTemplateDisableDownload = strings.ToLower(os.Getenv("NUCLEI_TEMPLATES_GITHUB_DISABLE_DOWNLOAD")) == "true"
options.GitLabTemplateDisableDownload = strings.ToLower(os.Getenv("NUCLEI_TEMPLATES_GITLAB_DISABLE_DOWNLOAD")) == "true"
options.AwsTemplateDisableDownload = strings.ToLower(os.Getenv("NUCLEI_TEMPLATES_AWS_DISABLE_DOWNLOAD")) == "true"
options.AzureTemplateDisableDownload = strings.ToLower(os.Getenv("NUCLEI_TEMPLATES_AZURE_DISABLE_DOWNLOAD")) == "true"
}
5 changes: 4 additions & 1 deletion v2/internal/runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,10 @@ func New(options *types.Options) (*Runner, error) {

// Check for template updates and update if available.
// If the custom templates manager is not nil, we will install custom templates if there is a fresh installation
tm := &installer.TemplateManager{CustomTemplates: ctm}
tm := &installer.TemplateManager{
CustomTemplates: ctm,
DisablePublicTemplates: options.PublicTemplateDisableDownload,
}
if err := tm.FreshInstallIfNotExists(); err != nil {
gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err)
}
Expand Down
2 changes: 1 addition & 1 deletion v2/pkg/catalog/config/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const (
NucleiTemplatesIndexFileName = ".templates-index" // contains index of official nuclei templates
NucleiTemplatesCheckSumFileName = ".checksum"
NewTemplateAdditionsFileName = ".new-additions"
CLIConifgFileName = "config.yaml"
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
Version = `v2.9.8`
Expand Down
2 changes: 1 addition & 1 deletion v2/pkg/catalog/config/nucleiconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func (c *Config) GetChecksumFilePath() string {

// GetCLIOptsConfigFilePath returns the nuclei cli config file path
func (c *Config) GetFlagsConfigFilePath() string {
return filepath.Join(c.configDir, CLIConifgFileName)
return filepath.Join(c.configDir, CLIConfigFileName)
}

// GetNewAdditions returns new template additions in current template release
Expand Down

0 comments on commit c672ee8

Please sign in to comment.