diff --git a/README.md b/README.md index 7007930a03..8c770a234a 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,9 @@ TEMPLATES: -ntv, -new-templates-version string[] run new templates added in specific version -as, -automatic-scan automatic web scan using wappalyzer technology detection to tags mapping -t, -templates string[] list of template or template directory to run (comma-separated, file) - -tu, -template-url string[] list of template urls to run (comma-separated, file) + -turl, -template-url string[] template url or list containing template urls to run (comma-separated, file) -w, -workflows string[] list of workflow or workflow directory to run (comma-separated, file) - -wu, -workflow-url string[] list of workflow urls to run (comma-separated, file) + -wurl, -workflow-url string[] workflow url or list containing workflow urls to run (comma-separated, file) -validate validate the passed templates to nuclei -nss, -no-strict-syntax disable strict syntax check on templates -td, -template-display displays the templates content diff --git a/docs/editor/ai.mdx b/docs/editor/ai.mdx index a5dee4172e..4824d138a6 100644 --- a/docs/editor/ai.mdx +++ b/docs/editor/ai.mdx @@ -6,14 +6,11 @@ title: 'AI Assistance' AI Prompt -[Nuclei Template Editor](https://templates.nuclei.sh/) employs AI to generate templates for vulnerability reports. This document seeks to guide you through the process, offering you usage tips and examples. +[Nuclei Template Editor](https://templates.nuclei.sh/) has AI to generate templates for vulnerability reports. This document helps to guide you through the process, offering you usage tips and examples. ## Overview -Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. It operates following these steps: - -1. **Interpret a Prompt**: Analyzes a detailed prompt outlining a specific vulnerability. -2. **Generate a Template**: Creates a Nuclei template using the AI API. +Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. First, the system interprets the user's prompt to identify a specific vulnerability. Then, it generates a template based on the steps required to reproduce the vulnerability along with all the necessary meta information to reproduce and remediate. --- diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index d0ba4aac0b..92b682ce7b 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error { } defer os.Remove("test-config.yaml") - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml") + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml") if err != nil { return err } @@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error { ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list") + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list") if err == nil { return fmt.Errorf("expected error for not allowed remote template list url") } @@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error { } defer os.Remove("test-config.yaml") - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml") + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml") if err != nil { return err } @@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404") + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } @@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404") + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 8edb9872bb..8c42eef256 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -137,14 +137,14 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), - flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", nil, "list of template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), - flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", nil, "list of workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"), flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), - flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"), + flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"templates.nuclei.sh"}, "allowed domain list to load remote templates from"), ) flagSet.CreateGroup("filters", "Filtering", diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 559448732f..f0d9cfe540 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -2,7 +2,6 @@ package runner import ( "bytes" - "os" "path/filepath" "strings" @@ -45,14 +44,12 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { if hasExtraFlags(r.options) { if r.options.TemplateDisplay { colorize := !r.options.NoColor - path := tpl.Path - tplBody, err := os.ReadFile(path) + tplBody, err := store.ReadTemplateFromURI(path, true) if err != nil { gologger.Error().Msgf("Could not read the template %s: %s", path, err) continue } - if colorize { path = aurora.Cyan(tpl.Path).String() tplBody, err = r.highlightTemplate(&tplBody) @@ -60,7 +57,6 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err) continue } - } gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody) } else { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 8535b82334..f633cbd7e0 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -1,8 +1,12 @@ package loader import ( + "fmt" + "io" + "net/url" "os" "sort" + "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -17,6 +21,15 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" +) + +const ( + httpPrefix = "http://" + httpsPrefix = "https://" ) // Config contains the configuration options for the loader @@ -120,6 +133,30 @@ func New(config *Config) (*Store, error) { finalWorkflows: config.Workflows, } + // Do a check to see if we have URLs in templates flag, if so + // we need to processs them separately and remove them from the initial list + var templatesFinal []string + for _, template := range config.Templates { + // TODO: Add and replace this with urlutil.IsURL() helper + if stringsutil.HasPrefixAny(template, httpPrefix, httpsPrefix) { + config.TemplateURLs = append(config.TemplateURLs, template) + } else { + templatesFinal = append(templatesFinal, template) + } + } + + // fix editor paths + remoteTemplates := []string{} + for _, v := range config.TemplateURLs { + if _, err := urlutil.Parse(v); err == nil { + remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v)) + } else { + templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file + } + } + config.TemplateURLs = remoteTemplates + store.finalTemplates = templatesFinal + urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 if urlBasedTemplatesProvided { remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs, config.RemoteTemplateDomainList) @@ -145,6 +182,43 @@ func New(config *Config) (*Store, error) { return store, nil } +func handleTemplatesEditorURLs(input string) string { + parsed, err := url.Parse(input) + if err != nil { + return input + } + if !strings.HasSuffix(parsed.Hostname(), "templates.nuclei.sh") { + return input + } + if strings.HasSuffix(parsed.Path, ".yaml") { + return input + } + parsed.Path = fmt.Sprintf("%s.yaml", parsed.Path) + finalURL := parsed.String() + return finalURL +} + +// ReadTemplateFromURI should only be used for viewing templates +// and should not be used anywhere else like loading and executing templates +// there is no sandbox restriction here +func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) { + if stringsutil.HasPrefixAny(uri, httpPrefix, httpsPrefix) && remote { + uri = handleTemplatesEditorURLs(uri) + remoteTemplates, _, err := getRemoteTemplatesAndWorkflows([]string{uri}, nil, store.config.RemoteTemplateDomainList) + if err != nil || len(remoteTemplates) == 0 { + return nil, errorutil.NewWithErr(err).Msgf("Could not load template %s: got %v", uri, remoteTemplates) + } + resp, err := retryablehttp.Get(remoteTemplates[0]) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(resp.Body) + } else { + return os.ReadFile(uri) + } +} + // Templates returns all the templates in the store func (store *Store) Templates() []*templates.Template { return store.templates diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index c5e0a64324..77bfddaa2b 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -56,7 +56,6 @@ func getRemoteTemplatesAndWorkflows(templateURLs, workflowURLs, remoteTemplateDo } } } - return remoteTemplateList, remoteWorkFlowList, err }