introduce template-encoded field

README.md

@@ -157,8 +157,9 @@ OUTPUT:
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
-   -irr, -include-rr             include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true)
+   -irr, -include-rr -omit-raw   include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true)
    -or, -omit-raw                omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
+   -ot, -omit-template           omit encoded template in the JSON, JSONL output
    -nm, -no-meta                 disable printing result metadata in cli output
    -ts, -timestamp               enables printing timestamp in cli output
    -rdb, -report-db string       nuclei reporting database (always use this to persist report data)

cmd/nuclei/main.go

@@ -222,6 +222,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"),
 		flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"),
+		flagSet.BoolVarP(&options.OmitTemplate, "omit-template", "ot", false, "omit encoded template in the JSON, JSONL output"),
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"),
 		flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"),

pkg/output/output.go

@@ -115,6 +115,8 @@ type ResultEvent struct {
 	TemplateID string `json:"template-id"`
 	// TemplatePath is the path of template
 	TemplatePath string `json:"template-path,omitempty"`
+	// TemplateEncoded is the base64 encoded template
+	TemplateEncoded string `json:"template-encoded,omitempty"`
 	// Info contains information block of the template for the result.
 	Info model.Info `json:"info,inline"`
 	// MatcherName is the name of the matcher matched if any.
@@ -217,6 +219,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 	if event.TemplatePath != "" {
 		event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID))
 	}
+
 	event.Timestamp = time.Now()
 
 	var data []byte

pkg/protocols/code/code.go

@@ -254,6 +254,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Timestamp:        time.Now(),
 		MatcherStatus:    true,
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/dns/operators.go

@@ -122,6 +122,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Timestamp:        time.Now(),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["raw"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/file/operators.go

@@ -107,6 +107,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Response:         types.ToString(wrapped.InternalEvent["raw"]),
 		Timestamp:        time.Now(),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/headless/operators.go

@@ -138,6 +138,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["data"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/http/operators.go

@@ -164,6 +164,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         request.truncateResponse(wrapped.InternalEvent["response"]),
 		CURLCommand:      types.ToString(wrapped.InternalEvent["curl-command"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/javascript/js.go

@@ -634,6 +634,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/network/operators.go

@@ -108,6 +108,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["data"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/offlinehttp/operators.go

@@ -151,6 +151,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["raw"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/protocols.go

@@ -1,6 +1,7 @@
 package protocols
 
 import (
+	"encoding/base64"
 	"sync/atomic"
 
 	"github.com/projectdiscovery/ratelimit"
@@ -30,6 +31,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
+var MaxTemplateFileSizeForEncoding = 1024 * 1024
+
 // Executer is an interface implemented any protocol based request executer.
 type Executer interface {
 	// Compile compiles the execution generators preparing any requests possible.
@@ -50,6 +53,8 @@ type ExecutorOptions struct {
 	TemplatePath string
 	// TemplateInfo contains information block of the template request
 	TemplateInfo model.Info
+	// RawTemplate is the raw template for the request
+	RawTemplate []byte
 	// Output is a writer interface for writing output events from executer.
 	Output output.Writer
 	// Options contains configuration options for the executer.
@@ -294,3 +299,10 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher
 	}
 	return false, nil
 }
+
+func (e *ExecutorOptions) EncodeTemplate() string {
+	if !e.Options.OmitTemplate && len(e.RawTemplate) <= MaxTemplateFileSizeForEncoding {
+		return base64.StdEncoding.EncodeToString(e.RawTemplate)
+	}
+	return ""
+}

pkg/protocols/ssl/ssl.go

@@ -375,6 +375,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Timestamp:        time.Now(),
 		MatcherStatus:    true,
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/websocket/websocket.go

@@ -409,6 +409,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		IP:               types.ToString(wrapped.InternalEvent["ip"]),
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/protocols/whois/whois.go

@@ -185,6 +185,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		MatcherStatus:    true,
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         types.ToString(wrapped.InternalEvent["response"]),
+		TemplateEncoded:  request.options.EncodeTemplate(),
 	}
 	return data
 }

pkg/templates/compile.go

@@ -128,7 +128,7 @@ func (template *Template) Requests() int {
 }
 
 // compileProtocolRequests compiles all the protocol requests for the template
-func (template *Template) compileProtocolRequests(options protocols.ExecutorOptions) error {
+func (template *Template) compileProtocolRequests(options *protocols.ExecutorOptions) error {
 	templateRequests := template.Requests()
 
 	if templateRequests == 0 {
@@ -180,7 +180,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti
 			requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsJavascript)...)
 		}
 	}
-	template.Executer = tmplexec.NewTemplateExecuter(requests, &options)
+	template.Executer = tmplexec.NewTemplateExecuter(requests, options)
 	return nil
 }
 
@@ -206,7 +206,7 @@ func (template *Template) convertRequestToProtocolsRequest(requests interface{})
 // compileOfflineHTTPRequest iterates all requests if offline http mode is
 // specified and collects all matchers for all the base request templates
 // (those with URL {{BaseURL}} and it's slash variation.)
-func (template *Template) compileOfflineHTTPRequest(options protocols.ExecutorOptions) error {
+func (template *Template) compileOfflineHTTPRequest(options *protocols.ExecutorOptions) error {
 	operatorsList := []*operators.Operators{}
 
 mainLoop:
@@ -225,7 +225,7 @@ mainLoop:
 	}
 	if len(operatorsList) > 0 {
 		options.Operators = operatorsList
-		template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
+		template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, options)
 		return nil
 	}
 
@@ -360,7 +360,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
 		return nil, errorutil.NewWithErr(err).Msgf("failed to load file refs for %s", template.ID)
 	}
 
-	if err := template.compileProtocolRequests(options); err != nil {
+	if err := template.compileProtocolRequests(template.Options); err != nil {
 		return nil, err
 	}
 
@@ -377,13 +377,18 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
 
 	// check if the template is verified
 	// only valid templates can be verified or signed
-	for _, verifier := range signer.DefaultTemplateVerifiers {
+	var verifier *signer.TemplateSigner
+	for _, verifier = range signer.DefaultTemplateVerifiers {
 		template.Verified, _ = verifier.Verify(data, template)
 		if template.Verified {
 			SignatureStats[verifier.Identifier()].Add(1)
 			break
 		}
 	}
+
+	if !(template.Verified && verifier.Identifier() == "projectdiscovery/nuclei-templates") {
+		template.Options.RawTemplate = data
+	}
 	return template, nil
 }
 

pkg/types/types.go

@@ -252,6 +252,8 @@ type Options struct {
 	JSONRequests bool
 	// OmitRawRequests omits requests/responses for matches in JSON output
 	OmitRawRequests bool
+	// OmitTemplate omits encoded template from JSON output
+	OmitTemplate bool
 	// JSONExport is the file to export JSON output format to
 	JSONExport string
 	// JSONLExport is the file to export JSONL output format to