From 33679f5b4fdbc6cb699a8488e1905ca841e5e8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 15 Nov 2023 10:36:06 +0000 Subject: [PATCH 1/8] introduce scan context --- pkg/core/executors.go | 20 ++--- pkg/core/workflow_execute.go | 114 ++++++++++++++--------------- pkg/output/output.go | 1 + pkg/protocols/protocols.go | 2 +- pkg/scan/scan_context.go | 52 +++++++++++++ pkg/templates/cluster.go | 11 ++- pkg/tmplexec/exec.go | 20 ++--- pkg/tmplexec/flow/flow_executor.go | 12 +-- pkg/tmplexec/generic/exec.go | 19 ++--- pkg/tmplexec/interface.go | 5 +- pkg/tmplexec/multiproto/multi.go | 23 +++--- 11 files changed, 169 insertions(+), 110 deletions(-) create mode 100644 pkg/scan/scan_context.go diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 262fdcefe7..56c311ae5d 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -5,7 +5,6 @@ import ( "sync/atomic" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -24,11 +23,12 @@ func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, res var err error var match bool if e.Callback != nil { - err = template.Executer.ExecuteWithResults(contextargs.New(), func(event *output.InternalWrappedEvent) { - for _, result := range event.Results { + if results, err := template.Executer.ExecuteWithResults(contextargs.New()); err != nil { + for _, result := range results { e.Callback(result) } - }) + } + match = true } else { match, err = template.Executer.Execute(contextargs.New()) @@ -118,11 +118,11 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target ctxArgs := contextargs.New() ctxArgs.MetaInput = value if e.Callback != nil { - err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) { - for _, result := range event.Results { + if results, err := template.Executer.ExecuteWithResults(ctxArgs); err != nil { + for _, result := range results { e.Callback(result) } - }) + } match = true } else { match, err = template.Executer.Execute(ctxArgs) @@ -173,11 +173,11 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta ctxArgs := contextargs.New() ctxArgs.MetaInput = value if e.Callback != nil { - err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) { - for _, result := range event.Results { + if results, err := template.Executer.ExecuteWithResults(ctxArgs); err != nil { + for _, result := range results { e.Callback(result) } - }) + } match = true } else { match, err = template.Executer.Execute(ctxArgs) diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go index a0e210587b..c65263ce15 100644 --- a/pkg/core/workflow_execute.go +++ b/pkg/core/workflow_execute.go @@ -1,14 +1,12 @@ package core import ( - "fmt" "net/http/cookiejar" "sync/atomic" "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" ) @@ -61,33 +59,33 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co // Don't print results with subtemplates, only print results on template. if len(template.Subtemplates) > 0 { - err = executer.Executer.ExecuteWithResults(input, func(result *output.InternalWrappedEvent) { - if result.OperatorsResult == nil { - return - } - if len(result.Results) > 0 { - firstMatched = true - } - - if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil { - for k, v := range result.OperatorsResult.Extracts { - // normalize items: - switch len(v) { - case 0, 1: - // - key:[item] => key: item - input.Set(k, v[0]) - default: - // - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n - for vIdx, vVal := range v { - normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx) - input.Set(normalizedKIdx, vVal) - } - // also add the original name with full slice - input.Set(k, v) - } - } - } - }) + // err = executer.Executer.ExecuteWithResults(input, func(result *output.InternalWrappedEvent) { + // if result.OperatorsResult == nil { + // return + // } + // if len(result.Results) > 0 { + // firstMatched = true + // } + + // if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil { + // for k, v := range result.OperatorsResult.Extracts { + // // normalize items: + // switch len(v) { + // case 0, 1: + // // - key:[item] => key: item + // input.Set(k, v[0]) + // default: + // // - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n + // for vIdx, vVal := range v { + // normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx) + // input.Set(normalizedKIdx, vVal) + // } + // // also add the original name with full slice + // input.Set(k, v) + // } + // } + // } + // }) } else { var matched bool matched, err = executer.Executer.Execute(input) @@ -115,35 +113,35 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co for _, executer := range template.Executers { executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) - err := executer.Executer.ExecuteWithResults(input, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - - if event.OperatorsResult.Extracts != nil { - for k, v := range event.OperatorsResult.Extracts { - input.Set(k, v) - } - } - - for _, matcher := range template.Matchers { - if !matcher.Match(event.OperatorsResult) { - continue - } - - for _, subtemplate := range matcher.Subtemplates { - swg.Add() - - go func(subtemplate *workflows.WorkflowTemplate) { - defer swg.Done() - - if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { - gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) - } - }(subtemplate) - } - } - }) + // err := executer.Executer.ExecuteWithResults(input, func(event *output.InternalWrappedEvent) { + // if event.OperatorsResult == nil { + // return + // } + + // if event.OperatorsResult.Extracts != nil { + // for k, v := range event.OperatorsResult.Extracts { + // input.Set(k, v) + // } + // } + + // for _, matcher := range template.Matchers { + // if !matcher.Match(event.OperatorsResult) { + // continue + // } + + // for _, subtemplate := range matcher.Subtemplates { + // swg.Add() + + // go func(subtemplate *workflows.WorkflowTemplate) { + // defer swg.Done() + + // if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { + // gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) + // } + // }(subtemplate) + // } + // } + // }) if err != nil { if len(template.Executers) == 1 { mainErr = err diff --git a/pkg/output/output.go b/pkg/output/output.go index f12594a4d8..dc298d0699 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -157,6 +157,7 @@ type ResultEvent struct { Lines []int `json:"matched-line,omitempty"` FileToIndexPosition map[string]int `json:"-"` + Error string `json:"error,omitempty"` } // NewStandardWriter creates a new output writer based on user configurations diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index c818b70685..81381468e9 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -42,7 +42,7 @@ type Executer interface { // Execute executes the protocol group and returns true or false if results were found. Execute(input *contextargs.Context) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input *contextargs.Context, callback OutputEventCallback) error + ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) } // ExecutorOptions contains the configuration options for executer clients diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go new file mode 100644 index 0000000000..03ec95c59d --- /dev/null +++ b/pkg/scan/scan_context.go @@ -0,0 +1,52 @@ +package scan + +import ( + "context" + + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" +) + +type ScanContext struct { + context.Context + ScanId string // MD5 (templateID+target+ip) + // existing Input/target related info + Input *contextargs.Context + // templateInfo + Info model.Info + // Globally shared args aka template Context + TemplateMap map[string]interface{} + // stats tracker like req count etc + // Stats *Stats + + errors []error + results []*output.ResultEvent + + OnError func(error) + OnResult func(e *output.InternalWrappedEvent) +} + +func NewScanContext(input *contextargs.Context) *ScanContext { + return &ScanContext{Input: input} +} + +func (s *ScanContext) GenerateResult() []*output.ResultEvent { + // ... + return s.results +} + +func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { + if s.OnResult != nil { + s.OnResult(e) + } + s.results = append(s.results, e.Results...) +} + +func (s *ScanContext) LogError(err error) error { + if s.OnError != nil { + s.OnError(err) + } + s.errors = append(s.errors, err) + return err +} diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index f98f1cd29e..6325931bb7 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" cryptoutil "github.com/projectdiscovery/utils/crypto" mapsutil "github.com/projectdiscovery/utils/maps" @@ -280,13 +281,14 @@ func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { +func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) { + scanCtx := scan.NewScanContext(input) dynamicValues := make(map[string]interface{}) inputItem := input.Clone() if e.options.InputHelper != nil && input.MetaInput.Input != "" { if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, e.templateType); input.MetaInput.Input == "" { - return nil + return nil, nil } } err := e.requests.ExecuteWithResults(inputItem, dynamicValues, nil, func(event *output.InternalWrappedEvent) { @@ -298,12 +300,13 @@ func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callbac event.InternalEvent["template-path"] = operator.templatePath event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) - callback(event) + // callback(event) + scanCtx.LogEvent(event) } } }) if err != nil && e.options.HostErrorsCache != nil { e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err) } - return err + return scanCtx.GenerateResult(), err } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 642906275b..0efce09cf3 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" @@ -111,7 +112,8 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { } } - cliExecutorCallback := func(event *output.InternalWrappedEvent) { + scanCtx := scan.NewScanContext(input) + scanCtx.OnResult = func(event *output.InternalWrappedEvent) { if event == nil { // something went wrong return @@ -142,9 +144,9 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { if err := flowexec.Compile(); err != nil { return false, err } - err = flowexec.ExecuteWithResults(input, cliExecutorCallback) + err = flowexec.ExecuteWithResults(scanCtx) } else { - err = e.engine.ExecuteWithResults(input, cliExecutorCallback) + err = e.engine.ExecuteWithResults(scanCtx) } if lastMatcherEvent != nil { @@ -154,11 +156,11 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { - userCallback := func(event *output.InternalWrappedEvent) { - if event != nil { - callback(event) - } +func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) { + scanCtx := scan.NewScanContext(input) + err := e.engine.ExecuteWithResults(scanCtx) + if err != nil && !e.options.Options.MatcherStatus { + return nil, err } - return e.engine.ExecuteWithResults(input, userCallback) + return scanCtx.GenerateResult(), err } diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 04394744c0..67a6c5c263 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -161,15 +162,15 @@ func (f *FlowExecutor) Compile() error { } // ExecuteWithResults executes the flow and returns results -func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { +func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { defer func() { if e := recover(); e != nil { - gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", input.MetaInput.Input, e) + gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e) panic(e) } }() - f.input = input + f.input = ctx.Input // -----Load all types of variables----- // add all input args to template context if f.input != nil && f.input.HasArgs() { @@ -177,7 +178,7 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value) }) } - if callback == nil { + if ctx.OnResult == nil { return fmt.Errorf("output callback cannot be nil") } // pass flow and execute the js vm and handle errors @@ -190,7 +191,8 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") } // this is where final result is generated/created - callback(f.lastEvent) + // callback(f.lastEvent) + ctx.LogEvent(f.lastEvent) if value.Export() != nil { f.results.Store(value.ToBoolean()) } else { diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index 022661cef1..254fdc107e 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -7,7 +7,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" ) // generic engine as name suggests is a generic template @@ -34,18 +34,18 @@ func (g *Generic) Compile() error { } // ExecuteWithResults executes the template and returns results -func (g *Generic) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { +func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) { + if ctx.Input.HasArgs() { + ctx.Input.ForEach(func(key string, value interface{}) { dynamicValues[key] = value }) } previous := make(map[string]interface{}) for _, req := range g.requests { - inputItem := input.Clone() - if g.options.InputHelper != nil && input.MetaInput.Input != "" { + inputItem := ctx.Input.Clone() + if g.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" { if inputItem.MetaInput.Input = g.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" { return nil } @@ -72,13 +72,14 @@ func (g *Generic) ExecuteWithResults(input *contextargs.Context, callback protoc } // for ExecuteWithResults : this callback will execute user defined callback and some error handling // for Execute : this callback will print the result to output - callback(event) + // callback(event) + ctx.LogEvent(event) }) if err != nil { if g.options.HostErrorsCache != nil { - g.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) + g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, input.MetaInput.PrettyPrint(), err) + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err) } // If a match was found and stop at first match is set, break out of the loop and return if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) { diff --git a/pkg/tmplexec/interface.go b/pkg/tmplexec/interface.go index d813e5bd2d..67f9621116 100644 --- a/pkg/tmplexec/interface.go +++ b/pkg/tmplexec/interface.go @@ -1,8 +1,7 @@ package tmplexec import ( - "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" @@ -26,7 +25,7 @@ type TemplateEngine interface { Compile() error // ExecuteWithResults executes the template and returns results - ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error + ExecuteWithResults(ctx *scan.ScanContext) error // Name returns name of template engine Name() string diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go index e9db400277..938c28d574 100644 --- a/pkg/tmplexec/multiproto/multi.go +++ b/pkg/tmplexec/multiproto/multi.go @@ -6,8 +6,8 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" ) // Mutliprotocol is a template executer engine that executes multiple protocols @@ -43,9 +43,9 @@ func (m *MultiProtocol) Compile() error { } // ExecuteWithResults executes the template and returns results -func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { +func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { // put all readonly args into template context - m.options.GetTemplateCtx(input.MetaInput).Merge(m.readOnlyArgs) + m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs) var finalProtoEvent *output.InternalWrappedEvent // callback to process results from all protocols multiProtoCallback := func(event *output.InternalWrappedEvent) { @@ -59,7 +59,7 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context) // currently if dynamic value array only contains one value we replace it with the value if len(v) == 1 { - m.options.GetTemplateCtx(input.MetaInput).Set(k, v[0]) + m.options.GetTemplateCtx(ctx.Input.MetaInput).Set(k, v[0]) } else { // Note: if extracted value contains multiple values then they can be accessed by indexing // ex: if values are dynamic = []string{"a","b","c"} then they are available as @@ -67,9 +67,9 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // we intentionally omit first index for unknown situations (where no of extracted values are not known) for i, val := range v { if i == 0 { - m.options.GetTemplateCtx(input.MetaInput).Set(k, val) + m.options.GetTemplateCtx(ctx.Input.MetaInput).Set(k, val) } else { - m.options.GetTemplateCtx(input.MetaInput).Set(k+strconv.Itoa(i), val) + m.options.GetTemplateCtx(ctx.Input.MetaInput).Set(k+strconv.Itoa(i), val) } } } @@ -77,8 +77,8 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback } // evaluate all variables after execution of each protocol - variableMap := m.options.Variables.Evaluate(m.options.GetTemplateCtx(input.MetaInput).GetAll()) - m.options.GetTemplateCtx(input.MetaInput).Merge(variableMap) // merge all variables into template context + variableMap := m.options.Variables.Evaluate(m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll()) + m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(variableMap) // merge all variables into template context } // template context: contains values extracted using `internal` extractor from previous protocols @@ -89,8 +89,8 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // execute all protocols in the queue for _, req := range m.requests { - values := m.options.GetTemplateCtx(input.MetaInput).GetAll() - err := req.ExecuteWithResults(input, output.InternalEvent(values), nil, multiProtoCallback) + values := m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll() + err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback) // if error skip execution of next protocols if err != nil { return err @@ -100,7 +100,8 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // currently the outer callback is only executed once (for the last protocol in queue) // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/pkg/protocols/common/executer/executem.go#L150 // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables - callback(finalProtoEvent) + // callback(finalProtoEvent) + ctx.LogEvent(finalProtoEvent) return nil } From f3447cc653d9df5866ac90207dbb54cfdf3f4319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 15 Nov 2023 10:57:15 +0000 Subject: [PATCH 2/8] minor --- pkg/core/workflow_execute_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/core/workflow_execute_test.go b/pkg/core/workflow_execute_test.go index fd9c09ed78..504d4d34e9 100644 --- a/pkg/core/workflow_execute_test.go +++ b/pkg/core/workflow_execute_test.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" "github.com/stretchr/testify/require" @@ -186,12 +187,13 @@ func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { +func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) { + scanCtx := scan.NewScanContext(input) if m.executeHook != nil { m.executeHook(input.MetaInput) } for _, output := range m.outputs { - callback(output) + scanCtx.LogEvent(output) } - return nil + return scanCtx.GenerateResult(), nil } From 722da28044acfaf972b7339a98e5978b53883bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 16 Nov 2023 12:57:00 +0000 Subject: [PATCH 3/8] add joined errors to resultevents --- pkg/scan/scan_context.go | 43 +++++++++++++++++++----------- pkg/templates/cluster.go | 1 - pkg/tmplexec/flow/flow_executor.go | 1 - pkg/tmplexec/generic/exec.go | 1 - pkg/tmplexec/multiproto/multi.go | 1 - 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 03ec95c59d..c75a2b464b 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -2,26 +2,17 @@ package scan import ( "context" + "strings" - "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" ) type ScanContext struct { context.Context - ScanId string // MD5 (templateID+target+ip) - // existing Input/target related info - Input *contextargs.Context - // templateInfo - Info model.Info - // Globally shared args aka template Context - TemplateMap map[string]interface{} - // stats tracker like req count etc - // Stats *Stats - - errors []error - results []*output.ResultEvent + Input *contextargs.Context + errors []error + events []*output.InternalWrappedEvent OnError func(error) OnResult func(e *output.InternalWrappedEvent) @@ -32,15 +23,35 @@ func NewScanContext(input *contextargs.Context) *ScanContext { } func (s *ScanContext) GenerateResult() []*output.ResultEvent { - // ... - return s.results + errorMessage := joinErrors(s.errors) + results := aggregateResults(s.events) + for _, result := range results { + result.Error = errorMessage + } + return results +} + +func aggregateResults(events []*output.InternalWrappedEvent) []*output.ResultEvent { + var results []*output.ResultEvent + for _, e := range events { + results = append(results, e.Results...) + } + return results +} + +func joinErrors(errors []error) string { + var errorMessages []string + for _, e := range errors { + errorMessages = append(errorMessages, e.Error()) + } + return strings.Join(errorMessages, "; ") } func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { if s.OnResult != nil { s.OnResult(e) } - s.results = append(s.results, e.Results...) + s.events = append(s.events, e) } func (s *ScanContext) LogError(err error) error { diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 6325931bb7..5310f84a03 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -300,7 +300,6 @@ func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context) ([]*out event.InternalEvent["template-path"] = operator.templatePath event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) - // callback(event) scanCtx.LogEvent(event) } } diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 67a6c5c263..4aba797850 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -191,7 +191,6 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") } // this is where final result is generated/created - // callback(f.lastEvent) ctx.LogEvent(f.lastEvent) if value.Export() != nil { f.results.Store(value.ToBoolean()) diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index 254fdc107e..606331e047 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -72,7 +72,6 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { } // for ExecuteWithResults : this callback will execute user defined callback and some error handling // for Execute : this callback will print the result to output - // callback(event) ctx.LogEvent(event) }) if err != nil { diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go index 938c28d574..dec575d2e8 100644 --- a/pkg/tmplexec/multiproto/multi.go +++ b/pkg/tmplexec/multiproto/multi.go @@ -100,7 +100,6 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { // currently the outer callback is only executed once (for the last protocol in queue) // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/pkg/protocols/common/executer/executem.go#L150 // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables - // callback(finalProtoEvent) ctx.LogEvent(finalProtoEvent) return nil From ead7ff9317eb19f62e16da77543fba67b8f5a774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 16 Nov 2023 13:19:32 +0000 Subject: [PATCH 4/8] change `executor` funcs' signature --- pkg/core/executors.go | 31 +++++---- pkg/core/workflow_execute.go | 131 ++++++++++++++++++----------------- pkg/protocols/protocols.go | 5 +- pkg/templates/cluster.go | 23 +++--- pkg/tmplexec/exec.go | 21 +++--- 5 files changed, 110 insertions(+), 101 deletions(-) diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 56c311ae5d..81baa020dc 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -6,6 +6,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -22,8 +23,9 @@ func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, res defer sg.Done() var err error var match bool + ctx := scan.NewScanContext(contextargs.New()) if e.Callback != nil { - if results, err := template.Executer.ExecuteWithResults(contextargs.New()); err != nil { + if results, err := template.Executer.ExecuteWithResults(ctx); err != nil { for _, result := range results { e.Callback(result) } @@ -31,7 +33,7 @@ func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, res match = true } else { - match, err = template.Executer.Execute(contextargs.New()) + match, err = template.Executer.Execute(ctx) } if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) @@ -111,21 +113,22 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target var match bool var err error + ctxArgs := contextargs.New() + ctxArgs.MetaInput = value + ctx := scan.NewScanContext(ctxArgs) switch template.Type() { case types.WorkflowProtocol: - match = e.executeWorkflow(value, template.CompiledWorkflow) + match = e.executeWorkflow(ctx, template.CompiledWorkflow) default: - ctxArgs := contextargs.New() - ctxArgs.MetaInput = value if e.Callback != nil { - if results, err := template.Executer.ExecuteWithResults(ctxArgs); err != nil { + if results, err := template.Executer.ExecuteWithResults(ctx); err != nil { for _, result := range results { e.Callback(result) } } match = true } else { - match, err = template.Executer.Execute(ctxArgs) + match, err = template.Executer.Execute(ctx) } } if err != nil { @@ -166,21 +169,22 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta var match bool var err error + ctxArgs := contextargs.New() + ctxArgs.MetaInput = value + ctx := scan.NewScanContext(ctxArgs) switch template.Type() { case types.WorkflowProtocol: - match = e.executeWorkflow(value, template.CompiledWorkflow) + match = e.executeWorkflow(ctx, template.CompiledWorkflow) default: - ctxArgs := contextargs.New() - ctxArgs.MetaInput = value if e.Callback != nil { - if results, err := template.Executer.ExecuteWithResults(ctxArgs); err != nil { + if results, err := template.Executer.ExecuteWithResults(ctx); err != nil { for _, result := range results { e.Callback(result) } } match = true } else { - match, err = template.Executer.Execute(ctxArgs) + match, err = template.Executer.Execute(ctx) } } if err != nil { @@ -221,7 +225,8 @@ func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs ctxArgs := contextargs.New() ctxArgs.MetaInput = value - match, err := template.Executer.Execute(ctxArgs) + ctx := scan.NewScanContext(ctxArgs) + match, err := template.Executer.Execute(ctx) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err) } diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go index c65263ce15..cb877cc60e 100644 --- a/pkg/core/workflow_execute.go +++ b/pkg/core/workflow_execute.go @@ -1,26 +1,29 @@ package core import ( + "fmt" "net/http/cookiejar" "sync/atomic" "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" ) const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" // executeWorkflow runs a workflow on an input and returns true or false -func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Workflow) bool { +func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) bool { results := &atomic.Bool{} // at this point we should be at the start root execution of a workflow tree, hence we create global shared instances workflowCookieJar, _ := cookiejar.New(nil) ctxArgs := contextargs.New() - ctxArgs.MetaInput = input + ctxArgs.MetaInput = ctx.Input.MetaInput ctxArgs.CookieJar = workflowCookieJar // we can know the nesting level only at runtime, so the best we can do here is increase template threads by one unit in case it's equal to 1 to allow @@ -37,7 +40,7 @@ func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Work func(template *workflows.WorkflowTemplate) { defer swg.Done() - if err := e.runWorkflowStep(template, ctxArgs, results, &swg, w); err != nil { + if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } }(template) @@ -48,7 +51,7 @@ func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Work // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *contextargs.Context, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { +func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { var firstMatched bool var err error var mainErr error @@ -59,43 +62,44 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co // Don't print results with subtemplates, only print results on template. if len(template.Subtemplates) > 0 { - // err = executer.Executer.ExecuteWithResults(input, func(result *output.InternalWrappedEvent) { - // if result.OperatorsResult == nil { - // return - // } - // if len(result.Results) > 0 { - // firstMatched = true - // } - - // if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil { - // for k, v := range result.OperatorsResult.Extracts { - // // normalize items: - // switch len(v) { - // case 0, 1: - // // - key:[item] => key: item - // input.Set(k, v[0]) - // default: - // // - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n - // for vIdx, vVal := range v { - // normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx) - // input.Set(normalizedKIdx, vVal) - // } - // // also add the original name with full slice - // input.Set(k, v) - // } - // } - // } - // }) + ctx.OnResult = func(result *output.InternalWrappedEvent) { + if result.OperatorsResult == nil { + return + } + if len(result.Results) > 0 { + firstMatched = true + } + + if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil { + for k, v := range result.OperatorsResult.Extracts { + // normalize items: + switch len(v) { + case 0, 1: + // - key:[item] => key: item + ctx.Input.Set(k, v[0]) + default: + // - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n + for vIdx, vVal := range v { + normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx) + ctx.Input.Set(normalizedKIdx, vVal) + } + // also add the original name with full slice + ctx.Input.Set(k, v) + } + } + } + } + _, err = executer.Executer.ExecuteWithResults(ctx) } else { var matched bool - matched, err = executer.Executer.Execute(input) + matched, err = executer.Executer.Execute(ctx) if matched { firstMatched = true } } if err != nil { if w.Options.HostErrorsCache != nil { - w.Options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) + w.Options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) } if len(template.Executers) == 1 { mainErr = err @@ -113,35 +117,36 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co for _, executer := range template.Executers { executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) - // err := executer.Executer.ExecuteWithResults(input, func(event *output.InternalWrappedEvent) { - // if event.OperatorsResult == nil { - // return - // } - - // if event.OperatorsResult.Extracts != nil { - // for k, v := range event.OperatorsResult.Extracts { - // input.Set(k, v) - // } - // } - - // for _, matcher := range template.Matchers { - // if !matcher.Match(event.OperatorsResult) { - // continue - // } - - // for _, subtemplate := range matcher.Subtemplates { - // swg.Add() - - // go func(subtemplate *workflows.WorkflowTemplate) { - // defer swg.Done() - - // if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { - // gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) - // } - // }(subtemplate) - // } - // } - // }) + ctx.OnResult = func(event *output.InternalWrappedEvent) { + if event.OperatorsResult == nil { + return + } + + if event.OperatorsResult.Extracts != nil { + for k, v := range event.OperatorsResult.Extracts { + ctx.Input.Set(k, v) + } + } + + for _, matcher := range template.Matchers { + if !matcher.Match(event.OperatorsResult) { + continue + } + + for _, subtemplate := range matcher.Subtemplates { + swg.Add() + + go func(subtemplate *workflows.WorkflowTemplate) { + defer swg.Done() + + if err := e.runWorkflowStep(subtemplate, ctx, results, swg, w); err != nil { + gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) + } + }(subtemplate) + } + } + } + _, err := executer.Executer.ExecuteWithResults(ctx) if err != nil { if len(template.Executers) == 1 { mainErr = err @@ -158,7 +163,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co swg.Add() go func(template *workflows.WorkflowTemplate) { - if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { + if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil { gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } swg.Done() diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 81381468e9..1e05080cab 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -27,6 +27,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" ) @@ -40,9 +41,9 @@ type Executer interface { // Requests returns the total number of requests the rule will perform Requests() int // Execute executes the protocol group and returns true or false if results were found. - Execute(input *contextargs.Context) (bool, error) + Execute(ctx *scan.ScanContext) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) + ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) } // ExecutorOptions contains the configuration options for executer clients diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 5310f84a03..fda477f083 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -10,7 +10,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -241,12 +240,12 @@ func (e *ClusterExecuter) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) { +func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { var results bool - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, e.templateType); input.MetaInput.Input == "" { + inputItem := ctx.Input.Clone() + if e.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" { + if inputItem.MetaInput.Input = e.options.InputHelper.Transform(ctx.Input.MetaInput.Input, e.templateType); ctx.Input.MetaInput.Input == "" { return false, nil } } @@ -275,19 +274,19 @@ func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) { } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err) + e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) } return results, err } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) { - scanCtx := scan.NewScanContext(input) +func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) { + scanCtx := scan.NewScanContext(ctx.Input) dynamicValues := make(map[string]interface{}) - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, e.templateType); input.MetaInput.Input == "" { + inputItem := ctx.Input.Clone() + if e.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" { + if inputItem.MetaInput.Input = e.options.InputHelper.Transform(ctx.Input.MetaInput.Input, e.templateType); ctx.Input.MetaInput.Input == "" { return nil, nil } } @@ -305,7 +304,7 @@ func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context) ([]*out } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err) + e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) } return scanCtx.GenerateResult(), err } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 0efce09cf3..a3d4a3b08a 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -94,12 +94,12 @@ func (e *TemplateExecuter) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { +func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { results := &atomic.Bool{} defer func() { // it is essential to remove template context of `Scan i.e template x input pair` // since it is of no use after scan is completed (regardless of success or failure) - e.options.RemoveTemplateCtx(input.MetaInput) + e.options.RemoveTemplateCtx(ctx.Input.MetaInput) }() var lastMatcherEvent *output.InternalWrappedEvent @@ -112,8 +112,7 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { } } - scanCtx := scan.NewScanContext(input) - scanCtx.OnResult = func(event *output.InternalWrappedEvent) { + ctx.OnResult = func(event *output.InternalWrappedEvent) { if event == nil { // something went wrong return @@ -140,13 +139,13 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { // so in compile step earlier we compile it to validate javascript syntax and other things // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { - flowexec := flow.NewFlowExecutor(e.requests, input, e.options, results) + flowexec := flow.NewFlowExecutor(e.requests, ctx.Input, e.options, results) if err := flowexec.Compile(); err != nil { return false, err } - err = flowexec.ExecuteWithResults(scanCtx) + err = flowexec.ExecuteWithResults(ctx) } else { - err = e.engine.ExecuteWithResults(scanCtx) + err = e.engine.ExecuteWithResults(ctx) } if lastMatcherEvent != nil { @@ -156,11 +155,11 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) { - scanCtx := scan.NewScanContext(input) - err := e.engine.ExecuteWithResults(scanCtx) +func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) { + // scanCtx := scan.NewScanContext(input) + err := e.engine.ExecuteWithResults(ctx) if err != nil && !e.options.Options.MatcherStatus { return nil, err } - return scanCtx.GenerateResult(), err + return ctx.GenerateResult(), err } From 0d923ec50312ef33e11aa2a53603aa74495def1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 16 Nov 2023 14:06:39 +0000 Subject: [PATCH 5/8] fix tests --- pkg/core/workflow_execute_test.go | 37 ++++++++++++++++--------- pkg/tmplexec/flow/flow_executor_test.go | 24 +++++++++++----- pkg/tmplexec/multiproto/multi_test.go | 9 ++++-- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/pkg/core/workflow_execute_test.go b/pkg/core/workflow_execute_test.go index 504d4d34e9..f3d6a7f231 100644 --- a/pkg/core/workflow_execute_test.go +++ b/pkg/core/workflow_execute_test.go @@ -25,7 +25,9 @@ func TestWorkflowsSimple(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) + input := contextargs.NewWithInput("https://test.com") + ctx := scan.NewScanContext(input) + matched := engine.executeWorkflow(ctx, workflow) require.True(t, matched, "could not get correct match value") } @@ -47,7 +49,9 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) + input := contextargs.NewWithInput("https://test.com") + ctx := scan.NewScanContext(input) + matched := engine.executeWorkflow(ctx, workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -73,7 +77,9 @@ func TestWorkflowsSubtemplates(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) + input := contextargs.NewWithInput("https://test.com") + ctx := scan.NewScanContext(input) + matched := engine.executeWorkflow(ctx, workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -97,7 +103,9 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) + input := contextargs.NewWithInput("https://test.com") + ctx := scan.NewScanContext(input) + matched := engine.executeWorkflow(ctx, workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -126,7 +134,9 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) + input := contextargs.NewWithInput("https://test.com") + ctx := scan.NewScanContext(input) + matched := engine.executeWorkflow(ctx, workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -155,7 +165,9 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) + input := contextargs.NewWithInput("https://test.com") + ctx := scan.NewScanContext(input) + matched := engine.executeWorkflow(ctx, workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -179,21 +191,20 @@ func (m *mockExecuter) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) { +func (m *mockExecuter) Execute(ctx *scan.ScanContext) (bool, error) { if m.executeHook != nil { - m.executeHook(input.MetaInput) + m.executeHook(ctx.Input.MetaInput) } return m.result, nil } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context) ([]*output.ResultEvent, error) { - scanCtx := scan.NewScanContext(input) +func (m *mockExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) { if m.executeHook != nil { - m.executeHook(input.MetaInput) + m.executeHook(ctx.Input.MetaInput) } for _, output := range m.outputs { - scanCtx.LogEvent(output) + ctx.LogEvent(output) } - return scanCtx.GenerateResult(), nil + return ctx.GenerateResult(), nil } diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index cb67c59fc2..e2053b423d 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/projectdiscovery/ratelimit" @@ -54,7 +55,8 @@ func TestFlowTemplateWithIndex(t *testing.T) { require.Nil(t, err, "could not compile template") input := contextargs.NewWithInput("hackerone.com") - gotresults, err := Template.Executer.Execute(input) + ctx := scan.NewScanContext(input) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) } @@ -72,7 +74,8 @@ func TestFlowTemplateWithID(t *testing.T) { require.Nil(t, err, "could not compile template") target := contextargs.NewWithInput("hackerone.com") - gotresults, err := Template.Executer.Execute(target) + ctx := scan.NewScanContext(target) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) } @@ -93,7 +96,8 @@ func TestFlowWithProtoPrefix(t *testing.T) { require.Nil(t, err, "could not compile template") input := contextargs.NewWithInput("hackerone.com") - gotresults, err := Template.Executer.Execute(input) + ctx := scan.NewScanContext(input) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) } @@ -112,8 +116,9 @@ func TestFlowWithConditionNegative(t *testing.T) { require.Nil(t, err, "could not compile template") input := contextargs.NewWithInput("scanme.sh") + ctx := scan.NewScanContext(input) // expect no results and verify thant dns request is executed and http is not - gotresults, err := Template.Executer.Execute(input) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.False(t, gotresults) } @@ -132,8 +137,9 @@ func TestFlowWithConditionPositive(t *testing.T) { require.Nil(t, err, "could not compile template") input := contextargs.NewWithInput("blog.projectdiscovery.io") + ctx := scan.NewScanContext(input) // positive match . expect results also verify that both dns() and http() were executed - gotresults, err := Template.Executer.Execute(input) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) } @@ -151,8 +157,10 @@ func TestFlowWithNoMatchers(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") + input := contextargs.NewWithInput("blog.projectdiscovery.io") + ctx := scan.NewScanContext(input) // positive match . expect results also verify that both dns() and http() were executed - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) @@ -165,8 +173,10 @@ func TestFlowWithNoMatchers(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") + anotherInput := contextargs.NewWithInput("blog.projectdiscovery.io") + anotherCtx := scan.NewScanContext(anotherInput) // positive match . expect results also verify that both dns() and http() were executed - gotresults, err = Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + gotresults, err = Template.Executer.Execute(anotherCtx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index 5dd1d65d42..cd260a049e 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/projectdiscovery/ratelimit" @@ -52,7 +53,9 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + input := contextargs.NewWithInput("blog.projectdiscovery.io") + ctx := scan.NewScanContext(input) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) } @@ -67,7 +70,9 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + input := contextargs.NewWithInput("blog.projectdiscovery.io") + ctx := scan.NewScanContext(input) + gotresults, err := Template.Executer.Execute(ctx) require.Nil(t, err, "could not execute template") require.True(t, gotresults) } From be8ad40fbbef2bd0592662decc2fcc351e0c69e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 17 Nov 2023 10:01:14 +0000 Subject: [PATCH 6/8] join errors in `LogError` func --- pkg/output/output.go | 1 + pkg/protocols/code/code.go | 1 + pkg/protocols/dns/operators.go | 1 + pkg/protocols/file/operators.go | 1 + pkg/protocols/headless/operators.go | 1 + pkg/protocols/http/operators.go | 1 + pkg/protocols/http/request.go | 31 +++++++++++++------------- pkg/protocols/javascript/js.go | 1 + pkg/protocols/network/operators.go | 1 + pkg/protocols/offlinehttp/operators.go | 1 + pkg/protocols/ssl/ssl.go | 1 + pkg/protocols/websocket/websocket.go | 1 + pkg/protocols/whois/whois.go | 1 + pkg/scan/scan_context.go | 17 +++++++++----- pkg/templates/cluster.go | 4 ++++ pkg/testutils/testutils.go | 1 + pkg/tmplexec/exec.go | 5 +---- pkg/tmplexec/flow/flow_executor.go | 2 ++ pkg/tmplexec/generic/exec.go | 1 + pkg/tmplexec/multiproto/multi.go | 1 + 20 files changed, 48 insertions(+), 26 deletions(-) diff --git a/pkg/output/output.go b/pkg/output/output.go index dc298d0699..e37f9decda 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -354,6 +354,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error Timestamp: time.Now(), //FIXME: this is workaround to encode the template when no results were found TemplateEncoded: w.encodeTemplate(types.ToString(event["template-path"])), + Error: types.ToString(event["error"]), } return w.Write(data) } diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 5977702e5e..bb8c06e549 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -255,6 +255,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Timestamp: time.Now(), MatcherStatus: true, TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go index 7aeafe5837..7b8467646b 100644 --- a/pkg/protocols/dns/operators.go +++ b/pkg/protocols/dns/operators.go @@ -123,6 +123,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index 347b5846c5..aee2abdfda 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -108,6 +108,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Response: types.ToString(wrapped.InternalEvent["raw"]), Timestamp: time.Now(), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go index c1bc0f2164..fd50ddaddb 100644 --- a/pkg/protocols/headless/operators.go +++ b/pkg/protocols/headless/operators.go @@ -139,6 +139,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 475bdf81b4..1e0f680399 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -165,6 +165,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Response: request.truncateResponse(wrapped.InternalEvent["response"]), CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index aac2bad71d..2328c8f30c 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -655,26 +655,25 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err) request.options.Progress.IncrementErrorsBy(1) - // If we have interactsh markers and request times out, still send + // In case of interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. - if hasInteractMatchers { - outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) - if i := strings.LastIndex(hostname, ":"); i != -1 { - hostname = hostname[:i] - } + // Also, to log failed use-cases. + outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) + if i := strings.LastIndex(hostname, ":"); i != -1 { + hostname = hostname[:i] + } - if input.MetaInput.CustomIP != "" { - outputEvent["ip"] = input.MetaInput.CustomIP - } else { - outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) - } + if input.MetaInput.CustomIP != "" { + outputEvent["ip"] = input.MetaInput.CustomIP + } else { + outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if request.CompiledOperators != nil { - event.InternalEvent = outputEvent - } - callback(event) + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + event.InternalEvent = outputEvent } + callback(event) return err } defer func() { diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 3736bf034c..d6c8917970 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -635,6 +635,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Response: types.ToString(wrapped.InternalEvent["response"]), IP: types.ToString(wrapped.InternalEvent["ip"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go index 74bee27980..8988496408 100644 --- a/pkg/protocols/network/operators.go +++ b/pkg/protocols/network/operators.go @@ -109,6 +109,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go index 0bd7a30ff9..0d906ff514 100644 --- a/pkg/protocols/offlinehttp/operators.go +++ b/pkg/protocols/offlinehttp/operators.go @@ -152,6 +152,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 63b7cfe77c..4fe556b3b9 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -376,6 +376,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index fdaebad356..4cedbc765e 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -410,6 +410,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index a7c02a9740..ce0bafe56c 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -186,6 +186,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), TemplateEncoded: request.options.EncodeTemplate(), + Error: types.ToString(wrapped.InternalEvent["error"]), } return data } diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index c75a2b464b..a7624b8ab9 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -23,12 +23,7 @@ func NewScanContext(input *contextargs.Context) *ScanContext { } func (s *ScanContext) GenerateResult() []*output.ResultEvent { - errorMessage := joinErrors(s.errors) - results := aggregateResults(s.events) - for _, result := range results { - result.Error = errorMessage - } - return results + return aggregateResults(s.events) } func aggregateResults(events []*output.InternalWrappedEvent) []*output.ResultEvent { @@ -59,5 +54,15 @@ func (s *ScanContext) LogError(err error) error { s.OnError(err) } s.errors = append(s.errors, err) + + errorMessage := joinErrors(s.errors) + results := aggregateResults(s.events) + for _, result := range results { + result.Error = errorMessage + } + for _, e := range s.events { + e.InternalEvent["error"] = errorMessage + } + return err } diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index fda477f083..e8eb435a8d 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -303,6 +303,10 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R } } }) + if err != nil { + ctx.LogError(err) + } + if err != nil && e.options.HostErrorsCache != nil { e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 3b916e5166..f212f935f7 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -180,6 +180,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven Timestamp: time.Now(), //FIXME: this is workaround to encode the template when no results were found TemplateEncoded: m.encodeTemplate(types.ToString(event["template-path"])), + Error: types.ToString(event["error"]), } return m.Write(data) } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index a3d4a3b08a..45f5058779 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -156,10 +156,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) { - // scanCtx := scan.NewScanContext(input) err := e.engine.ExecuteWithResults(ctx) - if err != nil && !e.options.Options.MatcherStatus { - return nil, err - } + ctx.LogError(err) return ctx.GenerateResult(), err } diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 4aba797850..690a4be991 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -184,10 +184,12 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { // pass flow and execute the js vm and handle errors value, err := f.jsVM.RunProgram(f.program) if err != nil { + ctx.LogError(err) return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) } runtimeErr := f.GetRuntimeErrors() if runtimeErr != nil { + ctx.LogError(runtimeErr) return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") } // this is where final result is generated/created diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index 606331e047..c5d25795cf 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -75,6 +75,7 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { ctx.LogEvent(event) }) if err != nil { + ctx.LogError(err) if g.options.HostErrorsCache != nil { g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) } diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go index dec575d2e8..021c83534d 100644 --- a/pkg/tmplexec/multiproto/multi.go +++ b/pkg/tmplexec/multiproto/multi.go @@ -93,6 +93,7 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback) // if error skip execution of next protocols if err != nil { + ctx.LogError(err) return err } } From 6554a98c27f4879094b0f3aa22cfbc96f3d6b112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 17 Nov 2023 10:03:46 +0000 Subject: [PATCH 7/8] change func signature --- pkg/scan/scan_context.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index a7624b8ab9..69f5712f73 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -49,7 +49,7 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { s.events = append(s.events, e) } -func (s *ScanContext) LogError(err error) error { +func (s *ScanContext) LogError(err error) { if s.OnError != nil { s.OnError(err) } @@ -63,6 +63,4 @@ func (s *ScanContext) LogError(err error) error { for _, e := range s.events { e.InternalEvent["error"] = errorMessage } - - return err } From d96db48fdb54b1d54167ebf4bbe41cd4addf2082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 17 Nov 2023 10:24:34 +0000 Subject: [PATCH 8/8] add guard --- pkg/scan/scan_context.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 69f5712f73..f1f73804b8 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -50,6 +50,10 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { } func (s *ScanContext) LogError(err error) { + if err == nil { + return + } + if s.OnError != nil { s.OnError(err) }