diff --git a/README.md b/README.md index 0bc63efd14..3cca8fb85c 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,9 @@ OUTPUT: -silent display findings only -nc, -no-color disable output content coloring (ANSI escape codes) -j, -jsonl write output in JSONL(ines) format - -irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true) + -irr, -include-rr -omit-raw include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use -omit-raw] (default true) -or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) + -ot, -omit-template omit encoded template in the JSON, JSONL output -nm, -no-meta disable printing result metadata in cli output -ts, -timestamp enables printing timestamp in cli output -rdb, -report-db string nuclei reporting database (always use this to persist report data) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index b16744dd9b..62d88759b5 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -75,12 +75,6 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) } defer reportingClient.Close() - outputWriter := testutils.NewMockOutputWriter() - var results []string - outputWriter.WriteCallback = func(event *output.ResultEvent) { - results = append(results, fmt.Sprintf("%v\n", event)) - } - defaultOpts := types.DefaultOptions() _ = protocolstate.Init(defaultOpts) _ = protocolinit.Init(defaultOpts) @@ -88,6 +82,12 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) defaultOpts.Templates = goflags.StringSlice{templatePath} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags + outputWriter := testutils.NewMockOutputWriter(defaultOpts.OmitTemplate) + var results []string + outputWriter.WriteCallback = func(event *output.ResultEvent) { + results = append(results, fmt.Sprintf("%v\n", event)) + } + interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) interactClient, err := interactsh.New(interactOpts) if err != nil { diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 539a5047fd..286fd03758 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -222,6 +222,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"), flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"), + flagSet.BoolVarP(&options.OmitTemplate, "omit-template", "ot", false, "omit encoded template in the JSON, JSONL output"), flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"), flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"), diff --git a/lib/sdk_private.go b/lib/sdk_private.go index cd84b0b6e7..4893da0de7 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -35,7 +35,7 @@ import ( // applyRequiredDefaults to options func (e *NucleiEngine) applyRequiredDefaults() { if e.customWriter == nil { - mockoutput := testutils.NewMockOutputWriter() + mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate) mockoutput.WriteCallback = func(event *output.ResultEvent) { if len(e.resultCallbacks) > 0 { for _, callback := range e.resultCallbacks { diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index c9a240229f..5ff7e8c5d9 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -51,6 +51,28 @@ type Config struct { configDir string `json:"-"` // Nuclei Global Config Directory } +// IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates. +// It checks if the template's path matches any of the predefined custom template directories +// (such as S3, GitHub, GitLab, and Azure directories). If the template resides in any of these directories, +// it is considered custom. Additionally, if the template's path does not start with the main Nuclei TemplatesDirectory, +// it is also considered custom. This function assumes that template paths are either absolute +// or relative to the same base as the paths configured in DefaultConfig. +func (c *Config) IsCustomTemplate(templatePath string) bool { + customDirs := []string{ + c.CustomS3TemplatesDirectory, + c.CustomGitHubTemplatesDirectory, + c.CustomGitLabTemplatesDirectory, + c.CustomAzureTemplatesDirectory, + } + + for _, dir := range customDirs { + if strings.HasPrefix(templatePath, dir) { + return true + } + } + return !strings.HasPrefix(templatePath, c.TemplatesDirectory) +} + // WriteVersionCheckData writes version check data to config file func (c *Config) WriteVersionCheckData(ignorehash, nucleiVersion, templatesVersion string) error { updated := false diff --git a/pkg/output/output.go b/pkg/output/output.go index 5f897f44dc..f12594a4d8 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -1,6 +1,7 @@ package output import ( + "encoding/base64" "fmt" "io" "os" @@ -20,6 +21,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/interactsh/pkg/server" "github.com/projectdiscovery/nuclei/v3/internal/colorizer" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/operators" @@ -60,6 +62,7 @@ type StandardWriter struct { severityColors func(severity.Severity) string storeResponse bool storeResponseDir string + omitTemplate bool } var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) @@ -115,6 +118,8 @@ type ResultEvent struct { TemplateID string `json:"template-id"` // TemplatePath is the path of template TemplatePath string `json:"template-path,omitempty"` + // TemplateEncoded is the base64 encoded template + TemplateEncoded string `json:"template-encoded,omitempty"` // Info contains information block of the template for the result. Info model.Info `json:"info,inline"` // MatcherName is the name of the matcher matched if any. @@ -207,6 +212,7 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { severityColors: colorizer.New(auroraColorizer), storeResponse: options.StoreResponse, storeResponseDir: options.StoreResponseDir, + omitTemplate: options.OmitTemplate, } return writer, nil } @@ -217,6 +223,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error { if event.TemplatePath != "" { event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID)) } + event.Timestamp = time.Now() var data []byte @@ -344,9 +351,22 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error Response: types.ToString(event["response"]), MatcherStatus: false, Timestamp: time.Now(), + //FIXME: this is workaround to encode the template when no results were found + TemplateEncoded: w.encodeTemplate(types.ToString(event["template-path"])), } return w.Write(data) } + +var maxTemplateFileSizeForEncoding = 1024 * 1024 + +func (w *StandardWriter) encodeTemplate(templatePath string) string { + data, err := os.ReadFile(templatePath) + if err == nil && !w.omitTemplate && len(data) <= maxTemplateFileSizeForEncoding && config.DefaultConfig.IsCustomTemplate(templatePath) { + return base64.StdEncoding.EncodeToString(data) + } + return "" +} + func sanitizeFileName(fileName string) string { fileName = strings.ReplaceAll(fileName, "http:", "") fileName = strings.ReplaceAll(fileName, "https:", "") diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index eccc6cd5fe..5977702e5e 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -254,6 +254,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), MatcherStatus: true, + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go index a4d57b2ec2..59570b22d1 100644 --- a/pkg/protocols/dns/operators.go +++ b/pkg/protocols/dns/operators.go @@ -122,6 +122,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Timestamp: time.Now(), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index 2780045b43..347b5846c5 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -107,6 +107,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Response: types.ToString(wrapped.InternalEvent["raw"]), Timestamp: time.Now(), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go index 6088f8f8b1..c1bc0f2164 100644 --- a/pkg/protocols/headless/operators.go +++ b/pkg/protocols/headless/operators.go @@ -138,6 +138,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index a5ee0efa8d..475bdf81b4 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -164,6 +164,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: request.truncateResponse(wrapped.InternalEvent["response"]), CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 4fb15cd08d..aeed664315 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -634,6 +634,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), IP: types.ToString(wrapped.InternalEvent["ip"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/javascript/js_test.go b/pkg/protocols/javascript/js_test.go index 2afd51a2aa..934b1fe7cb 100644 --- a/pkg/protocols/javascript/js_test.go +++ b/pkg/protocols/javascript/js_test.go @@ -32,7 +32,7 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go index 6e6c99ea53..74bee27980 100644 --- a/pkg/protocols/network/operators.go +++ b/pkg/protocols/network/operators.go @@ -108,6 +108,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go index 76d9fb77ab..0bd7a30ff9 100644 --- a/pkg/protocols/offlinehttp/operators.go +++ b/pkg/protocols/offlinehttp/operators.go @@ -151,6 +151,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 6acdbddfd7..c818b70685 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -1,6 +1,7 @@ package protocols import ( + "encoding/base64" "sync/atomic" "github.com/projectdiscovery/ratelimit" @@ -30,6 +31,8 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types" ) +var MaxTemplateFileSizeForEncoding = 1024 * 1024 + // Executer is an interface implemented any protocol based request executer. type Executer interface { // Compile compiles the execution generators preparing any requests possible. @@ -50,6 +53,8 @@ type ExecutorOptions struct { TemplatePath string // TemplateInfo contains information block of the template request TemplateInfo model.Info + // RawTemplate is the raw template for the request + RawTemplate []byte // Output is a writer interface for writing output events from executer. Output output.Writer // Options contains configuration options for the executer. @@ -294,3 +299,10 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher } return false, nil } + +func (e *ExecutorOptions) EncodeTemplate() string { + if !e.Options.OmitTemplate && len(e.RawTemplate) <= MaxTemplateFileSizeForEncoding { + return base64.StdEncoding.EncodeToString(e.RawTemplate) + } + return "" +} diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 5fc16f0777..63b7cfe77c 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -375,6 +375,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Timestamp: time.Now(), MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index a6da3b5d8e..fdaebad356 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -409,6 +409,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index df729b980a..a7c02a9740 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -185,6 +185,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent MatcherStatus: true, Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), + TemplateEncoded: request.options.EncodeTemplate(), } return data } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 5f5e634562..8d3f959a94 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -128,7 +128,7 @@ func (template *Template) Requests() int { } // compileProtocolRequests compiles all the protocol requests for the template -func (template *Template) compileProtocolRequests(options protocols.ExecutorOptions) error { +func (template *Template) compileProtocolRequests(options *protocols.ExecutorOptions) error { templateRequests := template.Requests() if templateRequests == 0 { @@ -180,7 +180,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsJavascript)...) } } - template.Executer = tmplexec.NewTemplateExecuter(requests, &options) + template.Executer = tmplexec.NewTemplateExecuter(requests, options) return nil } @@ -206,7 +206,7 @@ func (template *Template) convertRequestToProtocolsRequest(requests interface{}) // compileOfflineHTTPRequest iterates all requests if offline http mode is // specified and collects all matchers for all the base request templates // (those with URL {{BaseURL}} and it's slash variation.) -func (template *Template) compileOfflineHTTPRequest(options protocols.ExecutorOptions) error { +func (template *Template) compileOfflineHTTPRequest(options *protocols.ExecutorOptions) error { operatorsList := []*operators.Operators{} mainLoop: @@ -225,7 +225,7 @@ mainLoop: } if len(operatorsList) > 0 { options.Operators = operatorsList - template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, options) return nil } @@ -360,7 +360,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e return nil, errorutil.NewWithErr(err).Msgf("failed to load file refs for %s", template.ID) } - if err := template.compileProtocolRequests(options); err != nil { + if err := template.compileProtocolRequests(template.Options); err != nil { return nil, err } @@ -377,13 +377,18 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e // check if the template is verified // only valid templates can be verified or signed - for _, verifier := range signer.DefaultTemplateVerifiers { + var verifier *signer.TemplateSigner + for _, verifier = range signer.DefaultTemplateVerifiers { template.Verified, _ = verifier.Verify(data, template) if template.Verified { SignatureStats[verifier.Identifier()].Add(1) break } } + + if !(template.Verified && verifier.Identifier() == "projectdiscovery/nuclei-templates") { + template.Options.RawTemplate = data + } return template, nil } diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go index 05e5601ee3..8092dc4665 100644 --- a/pkg/templates/compile_test.go +++ b/pkg/templates/compile_test.go @@ -39,7 +39,7 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index da2d203053..3b916e5166 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -2,6 +2,8 @@ package testutils import ( "context" + "encoding/base64" + "os" "time" "github.com/projectdiscovery/ratelimit" @@ -84,7 +86,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco TemplateID: info.ID, TemplateInfo: info.Info, TemplatePath: info.Path, - Output: NewMockOutputWriter(), + Output: NewMockOutputWriter(options.OmitTemplate), Options: options, Progress: progressImpl, ProjectFile: nil, @@ -106,14 +108,15 @@ func (n *NoopWriter) Write(data []byte, level levels.Level) {} // MockOutputWriter is a mocked output writer. type MockOutputWriter struct { aurora aurora.Aurora + omitTemplate bool RequestCallback func(templateID, url, requestType string, err error) FailureCallback func(result *output.InternalEvent) WriteCallback func(o *output.ResultEvent) } // NewMockOutputWriter creates a new mock output writer -func NewMockOutputWriter() *MockOutputWriter { - return &MockOutputWriter{aurora: aurora.NewAurora(false)} +func NewMockOutputWriter(omomitTemplate bool) *MockOutputWriter { + return &MockOutputWriter{aurora: aurora.NewAurora(false), omitTemplate: omomitTemplate} } // Close closes the output writer interface @@ -175,10 +178,22 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven Response: types.ToString(event["response"]), MatcherStatus: false, Timestamp: time.Now(), + //FIXME: this is workaround to encode the template when no results were found + TemplateEncoded: m.encodeTemplate(types.ToString(event["template-path"])), } return m.Write(data) } +var maxTemplateFileSizeForEncoding = 1024 * 1024 + +func (w *MockOutputWriter) encodeTemplate(templatePath string) string { + data, err := os.ReadFile(templatePath) + if err == nil && !w.omitTemplate && len(data) <= maxTemplateFileSizeForEncoding && config.DefaultConfig.IsCustomTemplate(templatePath) { + return base64.StdEncoding.EncodeToString(data) + } + return "" +} + func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {} type MockProgressClient struct{} diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index f38514f977..cb67c59fc2 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -26,7 +26,7 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index c8ad42ca54..5dd1d65d42 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -26,7 +26,7 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/pkg/types/types.go b/pkg/types/types.go index 9e58f62800..2659bbd537 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -252,6 +252,8 @@ type Options struct { JSONRequests bool // OmitRawRequests omits requests/responses for matches in JSON output OmitRawRequests bool + // OmitTemplate omits encoded template from JSON output + OmitTemplate bool // JSONExport is the file to export JSON output format to JSONExport string // JSONLExport is the file to export JSONL output format to