introduce template-encoded field

README.md

@@ -157,8 +157,9 @@ OUTPUT:
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
-   -irr, -include-rr             include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true)
+   -irr, -include-rr -omit-raw   include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true)
    -or, -omit-raw                omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
+   -ot, -omit-template           omit encoded template in the JSON, JSONL output
    -nm, -no-meta                 disable printing result metadata in cli output
    -ts, -timestamp               enables printing timestamp in cli output
    -rdb, -report-db string       nuclei reporting database (always use this to persist report data)

cmd/integration-test/library.go

@@ -75,19 +75,19 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	}
 	defer reportingClient.Close()
 
-	outputWriter := testutils.NewMockOutputWriter()
-	var results []string
-	outputWriter.WriteCallback = func(event *output.ResultEvent) {
-		results = append(results, fmt.Sprintf("%v\n", event))
-	}
-
 	defaultOpts := types.DefaultOptions()
 	_ = protocolstate.Init(defaultOpts)
 	_ = protocolinit.Init(defaultOpts)
 
 	defaultOpts.Templates = goflags.StringSlice{templatePath}
 	defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
 
+	outputWriter := testutils.NewMockOutputWriter(defaultOpts.OmitTemplate)
+	var results []string
+	outputWriter.WriteCallback = func(event *output.ResultEvent) {
+		results = append(results, fmt.Sprintf("%v\n", event))
+	}
+
 	interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
 	interactClient, err := interactsh.New(interactOpts)
 	if err != nil {

cmd/nuclei/main.go

@@ -222,6 +222,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"),
 		flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"),
+		flagSet.BoolVarP(&options.OmitTemplate, "omit-template", "ot", false, "omit encoded template in the JSON, JSONL output"),
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"),
 		flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"),

lib/sdk_private.go

@@ -35,7 +35,7 @@ import (
 // applyRequiredDefaults to options
 func (e *NucleiEngine) applyRequiredDefaults() {
 	if e.customWriter == nil {
-		mockoutput := testutils.NewMockOutputWriter()
+		mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
 		mockoutput.WriteCallback = func(event *output.ResultEvent) {
 			if len(e.resultCallbacks) > 0 {
 				for _, callback := range e.resultCallbacks {

pkg/catalog/config/nucleiconfig.go

@@ -51,6 +51,28 @@ type Config struct {
 	configDir      string `json:"-"` //  Nuclei Global Config Directory
 }
 
+// IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates.
+// It checks if the template's path matches any of the predefined custom template directories
+// (such as S3, GitHub, GitLab, and Azure directories). If the template resides in any of these directories,
+// it is considered custom. Additionally, if the template's path does not start with the main Nuclei TemplatesDirectory,
+// it is also considered custom. This function assumes that template paths are either absolute
+// or relative to the same base as the paths configured in DefaultConfig.
+func (c *Config) IsCustomTemplate(templatePath string) bool {
+	customDirs := []string{
+		c.CustomS3TemplatesDirectory,
+		c.CustomGitHubTemplatesDirectory,
+		c.CustomGitLabTemplatesDirectory,
+		c.CustomAzureTemplatesDirectory,
+	}
+
+	for _, dir := range customDirs {
+		if strings.HasPrefix(templatePath, dir) {
+			return true
+		}
+	}
+	return !strings.HasPrefix(templatePath, c.TemplatesDirectory)
+}
+
 // WriteVersionCheckData writes version check data to config file
 func (c *Config) WriteVersionCheckData(ignorehash, nucleiVersion, templatesVersion string) error {
 	updated := false

pkg/output/output.go

@@ -1,6 +1,7 @@
 package output
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"os"
@@ -20,6 +21,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/interactsh/pkg/server"
 	"github.com/projectdiscovery/nuclei/v3/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
@@ -60,6 +62,7 @@ type StandardWriter struct {
 	severityColors   func(severity.Severity) string
 	storeResponse    bool
 	storeResponseDir string
+	omitTemplate     bool
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -115,6 +118,8 @@ type ResultEvent struct {
 	TemplateID string `json:"template-id"`
 	// TemplatePath is the path of template
 	TemplatePath string `json:"template-path,omitempty"`
+	// TemplateEncoded is the base64 encoded template
+	TemplateEncoded string `json:"template-encoded,omitempty"`
 	// Info contains information block of the template for the result.
 	Info model.Info `json:"info,inline"`
 	// MatcherName is the name of the matcher matched if any.
@@ -207,6 +212,7 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
 		severityColors:   colorizer.New(auroraColorizer),
 		storeResponse:    options.StoreResponse,
 		storeResponseDir: options.StoreResponseDir,
+		omitTemplate:     options.OmitTemplate,
 	}
 	return writer, nil
 }
@@ -217,6 +223,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 	if event.TemplatePath != "" {
 		event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID))
 	}
+
 	event.Timestamp = time.Now()
 
 	var data []byte
@@ -344,9 +351,22 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
 		Response:      types.ToString(event["response"]),
 		MatcherStatus: false,
 		Timestamp:     time.Now(),
+		//FIXME: this is workaround to encode the template when no results were found
+		TemplateEncoded: w.encodeTemplate(types.ToString(event["template-path"])),
 	}
 	return w.Write(data)
 }
+
+var maxTemplateFileSizeForEncoding = 1024 * 1024
+
+func (w *StandardWriter) encodeTemplate(templatePath string) string {
+	data, err := os.ReadFile(templatePath)
+	if err == nil && !w.omitTemplate && len(data) <= maxTemplateFileSizeForEncoding && config.DefaultConfig.IsCustomTemplate(templatePath) {
+		return base64.StdEncoding.EncodeToString(data)
+	}
+	return ""
+}
+
 func sanitizeFileName(fileName string) string {
 	fileName = strings.ReplaceAll(fileName, "http:", "")
 	fileName = strings.ReplaceAll(fileName, "https:", "")

pkg/protocols/code/code.go

@@ -254,6 +254,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Timestamp:        time.Now(),
 		MatcherStatus:    true,
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/dns/operators.go

@@ -122,6 +122,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Timestamp:        time.Now(),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["raw"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/file/operators.go

@@ -107,6 +107,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Response:         types.ToString(wrapped.InternalEvent["raw"]),
 		Timestamp:        time.Now(),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/headless/operators.go

@@ -138,6 +138,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["data"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/http/operators.go

@@ -164,6 +164,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         request.truncateResponse(wrapped.InternalEvent["response"]),
 		CURLCommand:      types.ToString(wrapped.InternalEvent["curl-command"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/javascript/js.go

@@ -634,6 +634,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/javascript/js_test.go

@@ -32,7 +32,7 @@ func setup() {
 	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	executerOpts = protocols.ExecutorOptions{
-		Output:       testutils.NewMockOutputWriter(),
+		Output:       testutils.NewMockOutputWriter(options.OmitTemplate),
 		Options:      options,
 		Progress:     progressImpl,
 		ProjectFile:  nil,

pkg/protocols/network/operators.go

@@ -108,6 +108,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["data"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/offlinehttp/operators.go

@@ -151,6 +151,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["raw"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/protocols.go

@@ -1,6 +1,7 @@
 package protocols
 
 import (
+	"encoding/base64"
 	"sync/atomic"
 
 	"github.com/projectdiscovery/ratelimit"
@@ -30,6 +31,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
+var MaxTemplateFileSizeForEncoding = 1024 * 1024
+
 // Executer is an interface implemented any protocol based request executer.
 type Executer interface {
 	// Compile compiles the execution generators preparing any requests possible.
@@ -50,6 +53,8 @@ type ExecutorOptions struct {
 	TemplatePath string
 	// TemplateInfo contains information block of the template request
 	TemplateInfo model.Info
+	// RawTemplate is the raw template for the request
+	RawTemplate []byte
 	// Output is a writer interface for writing output events from executer.
 	Output output.Writer
 	// Options contains configuration options for the executer.
@@ -294,3 +299,10 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher
 	}
 	return false, nil
 }
+
+func (e *ExecutorOptions) EncodeTemplate() string {
+	if !e.Options.OmitTemplate && len(e.RawTemplate) <= MaxTemplateFileSizeForEncoding {
+		return base64.StdEncoding.EncodeToString(e.RawTemplate)
+	}
+	return ""
+}

pkg/protocols/ssl/ssl.go

@@ -375,6 +375,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Timestamp:        time.Now(),
 		MatcherStatus:    true,
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/websocket/websocket.go

@@ -409,6 +409,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/whois/whois.go

@@ -185,6 +185,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		MatcherStatus:    true,
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/templates/compile.go

@@ -128,7 +128,7 @@ func (template *Template) Requests() int {
 }
 
 // compileProtocolRequests compiles all the protocol requests for the template
-func (template *Template) compileProtocolRequests(options protocols.ExecutorOptions) error {
+func (template *Template) compileProtocolRequests(options *protocols.ExecutorOptions) error {
 	templateRequests := template.Requests()
 
 	if templateRequests == 0 {
@@ -180,7 +180,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti
 			requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsJavascript)...)
 		}
 	}
-	template.Executer = tmplexec.NewTemplateExecuter(requests, &options)
+	template.Executer = tmplexec.NewTemplateExecuter(requests, options)
 	return nil
 }
 
@@ -206,7 +206,7 @@ func (template *Template) convertRequestToProtocolsRequest(requests interface{})
 // compileOfflineHTTPRequest iterates all requests if offline http mode is
 // specified and collects all matchers for all the base request templates
 // (those with URL {{BaseURL}} and it's slash variation.)
-func (template *Template) compileOfflineHTTPRequest(options protocols.ExecutorOptions) error {
+func (template *Template) compileOfflineHTTPRequest(options *protocols.ExecutorOptions) error {
 	operatorsList := []*operators.Operators{}
 
 mainLoop:
@@ -225,7 +225,7 @@ mainLoop:
 	}
 	if len(operatorsList) > 0 {
 		options.Operators = operatorsList
-		template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
+		template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, options)
 		return nil
 	}
 
@@ -360,7 +360,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
 		return nil, errorutil.NewWithErr(err).Msgf("failed to load file refs for %s", template.ID)
 	}
 
-	if err := template.compileProtocolRequests(options); err != nil {
+	if err := template.compileProtocolRequests(template.Options); err != nil {
 		return nil, err
 	}
 
@@ -377,13 +377,18 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
 
 	// check if the template is verified
 	// only valid templates can be verified or signed
-	for _, verifier := range signer.DefaultTemplateVerifiers {
+	var verifier *signer.TemplateSigner
+	for _, verifier = range signer.DefaultTemplateVerifiers {
 		template.Verified, _ = verifier.Verify(data, template)
 		if template.Verified {
 			SignatureStats[verifier.Identifier()].Add(1)
 			break
 		}
 	}
+
+	if !(template.Verified && verifier.Identifier() == "projectdiscovery/nuclei-templates") {
+		template.Options.RawTemplate = data
+	}
 	return template, nil
 }
 

pkg/templates/compile_test.go

@@ -39,7 +39,7 @@ func setup() {
 	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
 
 	executerOpts = protocols.ExecutorOptions{
-		Output:       testutils.NewMockOutputWriter(),
+		Output:       testutils.NewMockOutputWriter(options.OmitTemplate),
 		Options:      options,
 		Progress:     progressImpl,
 		ProjectFile:  nil,