From 3d2f31a56fc87ccae84afa21f4b4a537e1849e18 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Thu, 19 Sep 2024 18:58:20 +0530 Subject: [PATCH] fix missing template_url for pd signed templates when executed from custom path (#5644) --- pkg/catalog/loader/loader.go | 3 ++- pkg/keys/key.go | 2 ++ pkg/output/output.go | 5 +++-- pkg/protocols/code/code.go | 1 + pkg/protocols/dns/operators.go | 1 + pkg/protocols/file/operators.go | 1 + pkg/protocols/headless/operators.go | 1 + pkg/protocols/http/operators.go | 1 + pkg/protocols/javascript/js.go | 1 + pkg/protocols/network/operators.go | 15 ++++++++------- pkg/protocols/offlinehttp/operators.go | 1 + pkg/protocols/protocols.go | 2 ++ pkg/protocols/ssl/ssl.go | 1 + pkg/protocols/websocket/websocket.go | 1 + pkg/protocols/whois/whois.go | 1 + pkg/templates/compile.go | 5 ++--- pkg/templates/workflows.go | 3 ++- pkg/testutils/testutils.go | 2 +- pkg/utils/template_path.go | 16 +++++++--------- 19 files changed, 39 insertions(+), 24 deletions(-) diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index e1d8371e35..731ebbd68e 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" + "github.com/projectdiscovery/nuclei/v3/pkg/keys" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/templates" @@ -434,7 +435,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ // increment signed/unsigned counters if tmpl.Verified { if tmpl.TemplateVerifier == "" { - templates.SignatureStats[templates.PDVerifier].Add(1) + templates.SignatureStats[keys.PDVerifier].Add(1) } else { templates.SignatureStats[tmpl.TemplateVerifier].Add(1) } diff --git a/pkg/keys/key.go b/pkg/keys/key.go index bb0c900626..eac43f27ae 100644 --- a/pkg/keys/key.go +++ b/pkg/keys/key.go @@ -3,5 +3,7 @@ package keys import _ "embed" +const PDVerifier = "projectdiscovery/nuclei-templates" + //go:embed nuclei.crt var NucleiCert []byte // public key for verifying digital signature of templates diff --git a/pkg/output/output.go b/pkg/output/output.go index 4f02d71c38..07c6f4c71f 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -190,6 +190,7 @@ type ResultEvent struct { FuzzingPosition string `json:"fuzzing_position,omitempty"` FileToIndexPosition map[string]int `json:"-"` + TemplateVerifier string `json:"-"` Error string `json:"error,omitempty"` } @@ -263,7 +264,7 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { func (w *StandardWriter) Write(event *ResultEvent) error { // Enrich the result event with extra metadata on the template-path and url. if event.TemplatePath != "" { - event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID)) + event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier) } if len(w.KeysToRedact) > 0 { @@ -435,7 +436,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error // if no results were found, manually create a failure event event := wrappedEvent.InternalEvent - templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"])) + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"])) var templateInfo model.Info if event["template-info"] != nil { templateInfo = event["template-info"].(model.Info) diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 7736113dca..87a0128645 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -348,6 +348,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Matched: types.ToString(wrapped.InternalEvent["input"]), Host: fields.Host, diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go index 7b8467646b..0f18315303 100644 --- a/pkg/protocols/dns/operators.go +++ b/pkg/protocols/dns/operators.go @@ -114,6 +114,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index ff18af0979..5dab75bb91 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -104,6 +104,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go index d1b790f99c..3ad30e9c55 100644 --- a/pkg/protocols/headless/operators.go +++ b/pkg/protocols/headless/operators.go @@ -137,6 +137,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Path: fields.Path, diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 3de3e63e34..d630bfd8b0 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -170,6 +170,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index fbcd1a6ff9..0bd26b88a5 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -766,6 +766,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go index 4ff22130f4..2aa19e5b3b 100644 --- a/pkg/protocols/network/operators.go +++ b/pkg/protocols/network/operators.go @@ -100,13 +100,14 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent fields.Ip = types.ToString(wrapped.InternalEvent["ip"]) } data := &output.ResultEvent{ - TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), - TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), - Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: types.ToString(wrapped.InternalEvent["type"]), - Host: fields.Host, - Port: fields.Port, - URL: fields.URL, + TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), + Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, + Type: types.ToString(wrapped.InternalEvent["type"]), + Host: fields.Host, + Port: fields.Port, + URL: fields.URL, Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Metadata: wrapped.OperatorsResult.PayloadValues, diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go index 0d906ff514..f69abc4412 100644 --- a/pkg/protocols/offlinehttp/operators.go +++ b/pkg/protocols/offlinehttp/operators.go @@ -142,6 +142,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 8b8854cd8b..a9d50c1481 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -61,6 +61,8 @@ type ExecutorOptions struct { TemplatePath string // TemplateInfo contains information block of the template request TemplateInfo model.Info + // TemplateVerifier is the verifier for the template + TemplateVerifier string // RawTemplate is the raw template for the request RawTemplate []byte // Output is a writer interface for writing output events from executer. diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 50da53111b..681043d3bf 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -398,6 +398,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index a956de0e3f..aa099ef43a 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -402,6 +402,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: fields.Host, Port: fields.Port, diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index ce0bafe56c..9963ec19f8 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -177,6 +177,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, + TemplateVerifier: request.options.TemplateVerifier, Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 54708e7ea1..01af3a999b 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -36,8 +36,7 @@ var ( ) const ( - Unsigned = "unsigned" - PDVerifier = "projectdiscovery/nuclei-templates" + Unsigned = "unsigned" ) func init() { @@ -420,7 +419,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e break } } - + options.TemplateVerifier = template.TemplateVerifier if !(template.Verified && verifier.Identifier() == "projectdiscovery/nuclei-templates") { template.Options.RawTemplate = data } diff --git a/pkg/templates/workflows.go b/pkg/templates/workflows.go index bafb7aaf31..50029af18a 100644 --- a/pkg/templates/workflows.go +++ b/pkg/templates/workflows.go @@ -4,6 +4,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/keys" "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" @@ -105,7 +106,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr // increment signed/unsigned counters if template.Verified { if template.TemplateVerifier == "" { - SignatureStats[PDVerifier].Add(1) + SignatureStats[keys.PDVerifier].Add(1) } else { SignatureStats[template.TemplateVerifier].Add(1) } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 930787aab9..a1ed685c87 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -167,7 +167,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven // create event event := wrappedEvent.InternalEvent - templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"])) + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"])) var templateInfo model.Info if ti, ok := event["template-info"].(model.Info); ok { templateInfo = ti diff --git a/pkg/utils/template_path.go b/pkg/utils/template_path.go index f6ed8ae92c..6570d90f20 100644 --- a/pkg/utils/template_path.go +++ b/pkg/utils/template_path.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/keys" ) const ( @@ -12,16 +13,13 @@ const ( ) // TemplatePathURL returns the Path and URL for the provided template -func TemplatePathURL(fullPath, templateId string) (string, string) { - var templateDirectory string +func TemplatePathURL(fullPath, templateId, templateVerifier string) (path string, url string) { configData := config.DefaultConfig if configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { - templateDirectory = configData.TemplatesDirectory - } else { - return "", "" + path = strings.TrimPrefix(strings.TrimPrefix(fullPath, configData.TemplatesDirectory), "/") } - - finalPath := strings.TrimPrefix(strings.TrimPrefix(fullPath, templateDirectory), "/") - templateURL := TemplatesRepoURL + templateId - return finalPath, templateURL + if templateVerifier == keys.PDVerifier { + url = TemplatesRepoURL + templateId + } + return }