feat: added template-url support in template flag feature (#4089)

* misc docs update * feat: added template-url support in template flag feature * bugfix: added check URL condition * template domain update * editor host update * misc update * handle -turl template editor urls * view remote templates using -td * remove warning --------- Co-authored-by: sandeep <[email protected]> Co-authored-by: Tarun Koyalwar <[email protected]>
projectdiscovery · Aug 25, 2023 · 592a8a2 · 592a8a2
1 parent a2034ca
commit 592a8a2
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -126,9 +126,9 @@ TEMPLATES:
    -ntv, -new-templates-version string[]  run new templates added in specific version
    -as, -automatic-scan                   automatic web scan using wappalyzer technology detection to tags mapping
    -t, -templates string[]                list of template or template directory to run (comma-separated, file)
-   -tu, -template-url string[]            list of template urls to run (comma-separated, file)
+   -turl, -template-url string[]          template url or list containing template urls to run (comma-separated, file)
    -w, -workflows string[]                list of workflow or workflow directory to run (comma-separated, file)
-   -wu, -workflow-url string[]            list of workflow urls to run (comma-separated, file)
+   -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content

diff --git a/docs/editor/ai.mdx b/docs/editor/ai.mdx
@@ -6,14 +6,11 @@ title: 'AI Assistance'
 
 <img src="/images/ai.jpg" alt="AI Prompt" width="800px"/>
 
-[Nuclei Template Editor](https://templates.nuclei.sh/) employs AI to generate templates for vulnerability reports. This document seeks to guide you through the process, offering you usage tips and examples.
+[Nuclei Template Editor](https://templates.nuclei.sh/) has AI to generate templates for vulnerability reports. This document helps to guide you through the process, offering you usage tips and examples.
 
 ## Overview
 
-Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. It operates following these steps:
-
-1.  **Interpret a Prompt**: Analyzes a detailed prompt outlining a specific vulnerability.
-2.  **Generate a Template**: Creates a Nuclei template using the AI API.
+Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. First, the system interprets the user's prompt to identify a specific vulnerability. Then, it generates a template based on the steps required to reproduce the vulnerability along with all the necessary meta information to reproduce and remediate.
 
 ---
 

diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go
@@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
 	if err == nil {
 		return fmt.Errorf("expected error for not allowed remote template list url")
 	}
@@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
@@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
@@ -137,14 +137,14 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", nil, "list of template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", nil, "list of workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
-		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"),
+		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"templates.nuclei.sh"}, "allowed domain list to load remote templates from"),
 	)
 
 	flagSet.CreateGroup("filters", "Filtering",

diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go
@@ -2,7 +2,6 @@ package runner
 
 import (
 	"bytes"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -45,22 +44,19 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 		if hasExtraFlags(r.options) {
 			if r.options.TemplateDisplay {
 				colorize := !r.options.NoColor
-
 				path := tpl.Path
-				tplBody, err := os.ReadFile(path)
+				tplBody, err := store.ReadTemplateFromURI(path, true)
 				if err != nil {
 					gologger.Error().Msgf("Could not read the template %s: %s", path, err)
 					continue
 				}
-
 				if colorize {
 					path = aurora.Cyan(tpl.Path).String()
 					tplBody, err = r.highlightTemplate(&tplBody)
 					if err != nil {
 						gologger.Error().Msgf("Could not highlight the template %s: %s", tpl.Path, err)
 						continue
 					}
-
 				}
 				gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody)
 			} else {

diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go
@@ -1,8 +1,12 @@
 package loader
 
 import (
+	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
@@ -17,6 +21,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/retryablehttp-go"
+	errorutil "github.com/projectdiscovery/utils/errors"
+	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
+)
+
+const (
+	httpPrefix  = "http://"
+	httpsPrefix = "https://"
 )
 
 // Config contains the configuration options for the loader
@@ -120,6 +133,30 @@ func New(config *Config) (*Store, error) {
 		finalWorkflows: config.Workflows,
 	}
 
+	// Do a check to see if we have URLs in templates flag, if so
+	// we need to processs them separately and remove them from the initial list
+	var templatesFinal []string
+	for _, template := range config.Templates {
+		// TODO: Add and replace this with urlutil.IsURL() helper
+		if stringsutil.HasPrefixAny(template, httpPrefix, httpsPrefix) {
+			config.TemplateURLs = append(config.TemplateURLs, template)
+		} else {
+			templatesFinal = append(templatesFinal, template)
+		}
+	}
+
+	// fix editor paths
+	remoteTemplates := []string{}
+	for _, v := range config.TemplateURLs {
+		if _, err := urlutil.Parse(v); err == nil {
+			remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v))
+		} else {
+			templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
+		}
+	}
+	config.TemplateURLs = remoteTemplates
+	store.finalTemplates = templatesFinal
+
 	urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0
 	if urlBasedTemplatesProvided {
 		remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs, config.RemoteTemplateDomainList)
@@ -145,6 +182,43 @@ func New(config *Config) (*Store, error) {
 	return store, nil
 }
 
+func handleTemplatesEditorURLs(input string) string {
+	parsed, err := url.Parse(input)
+	if err != nil {
+		return input
+	}
+	if !strings.HasSuffix(parsed.Hostname(), "templates.nuclei.sh") {
+		return input
+	}
+	if strings.HasSuffix(parsed.Path, ".yaml") {
+		return input
+	}
+	parsed.Path = fmt.Sprintf("%s.yaml", parsed.Path)
+	finalURL := parsed.String()
+	return finalURL
+}
+
+// ReadTemplateFromURI should only be used for viewing templates
+// and should not be used anywhere else like loading and executing templates
+// there is no sandbox restriction here
+func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) {
+	if stringsutil.HasPrefixAny(uri, httpPrefix, httpsPrefix) && remote {
+		uri = handleTemplatesEditorURLs(uri)
+		remoteTemplates, _, err := getRemoteTemplatesAndWorkflows([]string{uri}, nil, store.config.RemoteTemplateDomainList)
+		if err != nil || len(remoteTemplates) == 0 {
+			return nil, errorutil.NewWithErr(err).Msgf("Could not load template %s: got %v", uri, remoteTemplates)
+		}
+		resp, err := retryablehttp.Get(remoteTemplates[0])
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		return io.ReadAll(resp.Body)
+	} else {
+		return os.ReadFile(uri)
+	}
+}
+
 // Templates returns all the templates in the store
 func (store *Store) Templates() []*templates.Template {
 	return store.templates

diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go
@@ -56,7 +56,6 @@ func getRemoteTemplatesAndWorkflows(templateURLs, workflowURLs, remoteTemplateDo
 			}
 		}
 	}
-
 	return remoteTemplateList, remoteWorkFlowList, err
 }