diff --git a/cmd/integration-test/interactsh.go b/cmd/integration-test/interactsh.go index d4cf328baa..536945fa44 100644 --- a/cmd/integration-test/interactsh.go +++ b/cmd/integration-test/interactsh.go @@ -1,11 +1,9 @@ package main -import osutils "github.com/projectdiscovery/utils/os" - // All Interactsh related testcases var interactshTestCases = []TestCaseInfo{ - {Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return true }}, - {Path: "protocols/http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return true }}, // disable this test for now + {Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return false }}, + {Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return false }}, // disable this test for now + {Path: "protocols/http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return false }}, {Path: "protocols/http/interactsh-requests-mc-and.yaml", TestCase: &httpInteractshRequestsWithMCAnd{}}, } diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index f28ac0f8fb..fd744c0579 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -319,6 +319,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"), + flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"), ) flagSet.CreateGroup("optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"), diff --git a/integration_tests/protocols/http/default-matcher-condition.yaml b/integration_tests/protocols/http/default-matcher-condition.yaml index b08acfc601..ea9d3b09b1 100644 --- a/integration_tests/protocols/http/default-matcher-condition.yaml +++ b/integration_tests/protocols/http/default-matcher-condition.yaml @@ -15,7 +15,7 @@ requests: - type: word part: interactsh_protocol words: - - "http" + - "dns" - type: status status: diff --git a/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml b/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml index b8063aadb0..a200ecdac9 100644 --- a/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml +++ b/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml @@ -24,6 +24,6 @@ requests: matchers: - type: word - part: interactsh_protocol # Confirms the HTTP Interaction + part: interactsh_protocol # Confirms DNS Interaction words: - - "http" \ No newline at end of file + - "dns" \ No newline at end of file diff --git a/integration_tests/protocols/http/interactsh.yaml b/integration_tests/protocols/http/interactsh.yaml index 28d9c56068..cd1892aee2 100644 --- a/integration_tests/protocols/http/interactsh.yaml +++ b/integration_tests/protocols/http/interactsh.yaml @@ -16,4 +16,4 @@ requests: - type: word part: interactsh_protocol # Confirms the HTTP Interaction words: - - "http" \ No newline at end of file + - "dns" \ No newline at end of file diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index f4af470ed1..fe204dee9f 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.1.7` + Version = `v3.1.8-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 1233943358..d601b51220 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -3,78 +3,34 @@ package compiler import ( "context" - "runtime/debug" + "fmt" "time" "github.com/dop251/goja" - "github.com/dop251/goja/parser" - "github.com/dop251/goja_nodejs/console" - "github.com/dop251/goja_nodejs/require" - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" - - "github.com/projectdiscovery/gologger" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libldap" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmssql" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmysql" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libnet" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/liboracle" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpop3" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpostgres" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librdp" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libredis" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librsync" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmb" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmtp" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libssh" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libstructs" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libtelnet" - _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libvnc" - "github.com/projectdiscovery/nuclei/v3/pkg/js/global" - "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" contextutil "github.com/projectdiscovery/utils/context" ) // Compiler provides a runtime to execute goja runtime // based javascript scripts efficiently while also // providing them access to custom modules defined in libs/. -type Compiler struct { - registry *require.Registry -} +type Compiler struct{} // New creates a new compiler for the goja runtime. func New() *Compiler { - registry := new(require.Registry) // this can be shared by multiple runtimes - // autoregister console node module with default printer it uses gologger backend - require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter())) - return &Compiler{registry: registry} + return &Compiler{} } // ExecuteOptions provides options for executing a script. type ExecuteOptions struct { - // Pool specifies whether to use a pool of goja runtimes - // Can be used to speedup execution but requires - // the script to not make any global changes. - Pool bool - - // CaptureOutput specifies whether to capture the output - // of the script execution. - CaptureOutput bool - - // CaptureVariables specifies the variables to capture - // from the script execution. - CaptureVariables []string - // Callback can be used to register new runtime helper functions // ex: export etc Callback func(runtime *goja.Runtime) error + // Cleanup is extra cleanup function to be called after execution + Cleanup func(runtime *goja.Runtime) + /// Timeout for this script execution Timeout int } @@ -111,51 +67,30 @@ func (e ExecuteResult) GetSuccess() bool { // Execute executes a script with the default options. func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) { - return c.ExecuteWithOptions(code, args, &ExecuteOptions{}) -} - -// VM returns a new goja runtime for the compiler. -func (c *Compiler) VM() *goja.Runtime { - runtime := c.newRuntime(false) - runtime.SetParserOptions(parser.WithDisableSourceMaps) - c.registerHelpersForVM(runtime) - return runtime + p, err := goja.Compile("", code, false) + if err != nil { + return nil, err + } + return c.ExecuteWithOptions(p, args, &ExecuteOptions{}) } // ExecuteWithOptions executes a script with the provided options. -func (c *Compiler) ExecuteWithOptions(code string, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) { - defer func() { - if err := recover(); err != nil { - gologger.Error().Msgf("Recovered panic %s %v: %v", code, args, err) - gologger.Verbose().Msgf("%s", debug.Stack()) - return - } - }() +func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) { if opts == nil { opts = &ExecuteOptions{} } - runtime := c.newRuntime(opts.Pool) - c.registerHelpersForVM(runtime) - - // register runtime functions if any - if opts.Callback != nil { - if err := opts.Callback(runtime); err != nil { - return nil, err - } - } - if args == nil { args = NewExecuteArgs() } - for k, v := range args.Args { - _ = runtime.Set(k, v) - } + // handle nil maps if args.TemplateCtx == nil { args.TemplateCtx = make(map[string]interface{}) } + if args.Args == nil { + args.Args = make(map[string]interface{}) + } // merge all args into templatectx args.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args) - _ = runtime.Set("template", args.TemplateCtx) if opts.Timeout <= 0 || opts.Timeout > 180 { // some js scripts can take longer time so allow configuring timeout @@ -170,72 +105,13 @@ func (c *Compiler) ExecuteWithOptions(code string, args *ExecuteArgs, opts *Exec results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { defer func() { if r := recover(); r != nil { - err = errors.Errorf("panic: %v", r) + err = fmt.Errorf("panic: %v", r) } }() - return runtime.RunString(code) + return executeProgram(program, args, opts) }) if err != nil { return nil, err } - captured := results.Export() - - if opts.CaptureOutput { - return convertOutputToResult(captured) - } - if len(opts.CaptureVariables) > 0 { - return c.captureVariables(runtime, opts.CaptureVariables) - } - // success is true by default . since js throws errors on failure - // hence output result is always success - return ExecuteResult{"response": captured, "success": results.ToBoolean()}, nil -} - -// captureVariables captures the variables from the runtime. -func (c *Compiler) captureVariables(runtime *goja.Runtime, variables []string) (ExecuteResult, error) { - results := make(ExecuteResult, len(variables)) - for _, variable := range variables { - value := runtime.Get(variable) - if value == nil { - continue - } - results[variable] = value.Export() - } - return results, nil -} - -func convertOutputToResult(output interface{}) (ExecuteResult, error) { - marshalled, err := jsoniter.Marshal(output) - if err != nil { - return nil, errors.Wrap(err, "could not marshal output") - } - - var outputMap map[string]interface{} - if err := jsoniter.Unmarshal(marshalled, &outputMap); err != nil { - var v interface{} - if unmarshalErr := jsoniter.Unmarshal(marshalled, &v); unmarshalErr != nil { - return nil, unmarshalErr - } - outputMap = map[string]interface{}{"output": v} - return outputMap, nil - } - return outputMap, nil -} - -// newRuntime creates a new goja runtime -// TODO: Add support for runtime reuse for helper functions -func (c *Compiler) newRuntime(reuse bool) *goja.Runtime { - return protocolstate.NewJSRuntime() -} - -// registerHelpersForVM registers all the helper functions for the goja runtime. -func (c *Compiler) registerHelpersForVM(runtime *goja.Runtime) { - _ = c.registry.Enable(runtime) - // by default import below modules every time - _ = runtime.Set("console", require.Require(runtime, console.ModuleName)) - - // Register embedded scripts - if err := global.RegisterNativeScripts(runtime); err != nil { - gologger.Error().Msgf("Could not register scripts: %s\n", err) - } + return ExecuteResult{"response": results.Export(), "success": results.ToBoolean()}, nil } diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go index 8a85a75e33..ca09e7782e 100644 --- a/pkg/js/compiler/compiler_test.go +++ b/pkg/js/compiler/compiler_test.go @@ -38,36 +38,6 @@ func TestExecuteResultGetSuccess(t *testing.T) { } } -func TestCompilerCaptureVariables(t *testing.T) { - compiler := New() - result, err := compiler.ExecuteWithOptions("var a = 1;", NewExecuteArgs(), &ExecuteOptions{CaptureVariables: []string{"a"}}) - if err != nil { - t.Fatal(err) - } - gotValue, ok := result["a"] - if !ok { - t.Fatalf("expected a to be present in the result") - } - if gotValue.(int64) != 1 { - t.Fatalf("expected a to be 1, got=%v", gotValue) - } -} - -func TestCompilerCaptureOutput(t *testing.T) { - compiler := New() - result, err := compiler.ExecuteWithOptions("let obj = {'a':'b'}; obj", NewExecuteArgs(), &ExecuteOptions{CaptureOutput: true}) - if err != nil { - t.Fatal(err) - } - gotValue, ok := result["a"] - if !ok { - t.Fatalf("expected a to be present in the result") - } - if gotValue.(string) != "b" { - t.Fatalf("expected a to be b, got=%v", gotValue) - } -} - type noopWriter struct { Callback func(data []byte, level levels.Level) } diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go index ed7d2e8f86..87f319c565 100644 --- a/pkg/js/compiler/init.go +++ b/pkg/js/compiler/init.go @@ -7,6 +7,7 @@ import "github.com/projectdiscovery/nuclei/v3/pkg/types" var ( // Per Execution Javascript timeout in seconds JsProtocolTimeout = 10 + JsVmConcurrency = 500 ) // Init initializes the javascript protocol @@ -15,6 +16,11 @@ func Init(opts *types.Options) error { // keep existing 10s timeout return nil } + if opts.JsConcurrency < 100 { + // 100 is reasonable default + opts.JsConcurrency = 100 + } JsProtocolTimeout = opts.Timeout + JsVmConcurrency = opts.JsConcurrency return nil } diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go new file mode 100644 index 0000000000..5d600e68a4 --- /dev/null +++ b/pkg/js/compiler/pool.go @@ -0,0 +1,116 @@ +package compiler + +import ( + "fmt" + "sync" + + "github.com/dop251/goja" + "github.com/dop251/goja_nodejs/console" + "github.com/dop251/goja_nodejs/require" + "github.com/projectdiscovery/gologger" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libldap" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmssql" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libmysql" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libnet" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/liboracle" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpop3" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libpostgres" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librdp" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libredis" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/librsync" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmb" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libsmtp" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libssh" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libstructs" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libtelnet" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libvnc" + "github.com/projectdiscovery/nuclei/v3/pkg/js/global" + "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" + "github.com/remeh/sizedwaitgroup" +) + +var ( + r *require.Registry + lazyRegistryInit = sync.OnceFunc(func() { + r = new(require.Registry) // this can be shared by multiple runtimes + // autoregister console node module with default printer it uses gologger backend + require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter())) + }) + sg sizedwaitgroup.SizedWaitGroup + lazySgInit = sync.OnceFunc(func() { + sg = sizedwaitgroup.New(JsVmConcurrency) + }) +) + +func getRegistry() *require.Registry { + lazyRegistryInit() + return r +} + +var gojapool = &sync.Pool{ + New: func() interface{} { + runtime := protocolstate.NewJSRuntime() + _ = getRegistry().Enable(runtime) + // by default import below modules every time + _ = runtime.Set("console", require.Require(runtime, console.ModuleName)) + + // Register embedded javacript helpers + if err := global.RegisterNativeScripts(runtime); err != nil { + gologger.Error().Msgf("Could not register scripts: %s\n", err) + } + return runtime + }, +} + +// executes the actual js program +func executeProgram(p *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (result goja.Value, err error) { + // its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static + // unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones + lazySgInit() + sg.Add() + defer sg.Done() + runtime := gojapool.Get().(*goja.Runtime) + defer func() { + // reset before putting back to pool + _ = runtime.GlobalObject().Delete("template") // template ctx + // remove all args + for k := range args.Args { + _ = runtime.GlobalObject().Delete(k) + } + if opts != nil && opts.Cleanup != nil { + opts.Cleanup(runtime) + } + gojapool.Put(runtime) + }() + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %s", r) + } + }() + // set template ctx + _ = runtime.Set("template", args.TemplateCtx) + // set args + for k, v := range args.Args { + _ = runtime.Set(k, v) + } + // register extra callbacks if any + if opts != nil && opts.Callback != nil { + if err := opts.Callback(runtime); err != nil { + return nil, err + } + + } + // execute the script + return runtime.RunProgram(p) +} + +// Internal purposes i.e generating bindings +func InternalGetGeneratorRuntime() *goja.Runtime { + runtime := gojapool.Get().(*goja.Runtime) + return runtime +} diff --git a/pkg/js/devtools/bindgen/generator.go b/pkg/js/devtools/bindgen/generator.go index 9218124066..8c471e35a2 100644 --- a/pkg/js/devtools/bindgen/generator.go +++ b/pkg/js/devtools/bindgen/generator.go @@ -150,8 +150,7 @@ func CreateTemplateData(directory string, packagePrefix string) (*TemplateData, // InitNativeScripts initializes the native scripts array // with all the exported functions from the runtime func (d *TemplateData) InitNativeScripts() { - compiler := compiler.New() - runtime := compiler.VM() + runtime := compiler.InternalGetGeneratorRuntime() exports := runtime.Get("exports") if exports == nil { diff --git a/pkg/protocols/http/httputils/chain.go b/pkg/protocols/http/httputils/chain.go new file mode 100644 index 0000000000..6ff9097922 --- /dev/null +++ b/pkg/protocols/http/httputils/chain.go @@ -0,0 +1,169 @@ +package httputils + +import ( + "bytes" + "fmt" + "net/http" + "sync" + + protoUtil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" +) + +// use buffer pool for storing response body +// and reuse it for each request +var bufPool = sync.Pool{ + New: func() any { + // The Pool's New function should generally only return pointer + // types, since a pointer can be put into the return interface + // value without an allocation: + return new(bytes.Buffer) + }, +} + +// getBuffer returns a buffer from the pool +func getBuffer() *bytes.Buffer { + return bufPool.Get().(*bytes.Buffer) +} + +// putBuffer returns a buffer to the pool +func putBuffer(buf *bytes.Buffer) { + buf.Reset() + bufPool.Put(buf) +} + +// Performance Notes: +// do not use http.Response once we create ResponseChain from it +// as this reuses buffers and saves allocations and also drains response +// body automatically. +// In required cases it can be used but should never be used for anything +// related to response body. +// Bytes.Buffer returned by getters should not be used and are only meant for convinience +// purposes like .String() or .Bytes() calls. +// Remember to call Close() on ResponseChain once you are done with it. + +// ResponseChain is a response chain for a http request +// on every call to previous it returns the previous response +// if it was redirected. +type ResponseChain struct { + headers *bytes.Buffer + body *bytes.Buffer + fullResponse *bytes.Buffer + resp *http.Response + reloaded bool // if response was reloaded to its previous redirect +} + +// NewResponseChain creates a new response chain for a http request +// with a maximum body size. (if -1 stick to default 4MB) +func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain { + if _, ok := resp.Body.(protoUtil.LimitResponseBody); !ok { + resp.Body = protoUtil.NewLimitResponseBodyWithSize(resp.Body, maxBody) + } + return &ResponseChain{ + headers: getBuffer(), + body: getBuffer(), + fullResponse: getBuffer(), + resp: resp, + } +} + +// Response returns the current response in the chain +func (r *ResponseChain) Headers() *bytes.Buffer { + return r.headers +} + +// Body returns the current response body in the chain +func (r *ResponseChain) Body() *bytes.Buffer { + return r.body +} + +// FullResponse returns the current response in the chain +func (r *ResponseChain) FullResponse() *bytes.Buffer { + return r.fullResponse +} + +// previous updates response pointer to previous response +// if it was redirected and returns true else false +func (r *ResponseChain) Previous() bool { + if r.resp != nil && r.resp.Request != nil && r.resp.Request.Response != nil { + r.resp = r.resp.Request.Response + r.reloaded = true + return true + } + return false +} + +// Fill buffers +func (r *ResponseChain) Fill() error { + r.reset() + if r.resp == nil { + return fmt.Errorf("response is nil") + } + + // load headers + err := DumpResponseIntoBuffer(r.resp, false, r.headers) + if err != nil { + return fmt.Errorf("error dumping response headers: %s", err) + } + + if r.resp.StatusCode != http.StatusSwitchingProtocols && !r.reloaded { + // Note about reloaded: + // this is a known behaviour existing from earlier version + // when redirect is followed and operators are executed on all redirect chain + // body of those requests is not available since its already been redirected + // This is not a issue since redirect happens with empty body according to RFC + // but this may be required sometimes + // Solution: Manual redirect using dynamic matchers or hijack redirected responses + // at transport level at replace with bytes buffer and then use it + + // load body + err = readNNormalizeRespBody(r, r.body) + if err != nil { + return fmt.Errorf("error reading response body: %s", err) + } + + // response body should not be used anymore + // drain and close + DrainResponseBody(r.resp) + } + + // join headers and body + r.fullResponse.Write(r.headers.Bytes()) + r.fullResponse.Write(r.body.Bytes()) + return nil +} + +// Close the response chain and releases the buffers. +func (r *ResponseChain) Close() { + putBuffer(r.headers) + putBuffer(r.body) + putBuffer(r.fullResponse) + r.headers = nil + r.body = nil + r.fullResponse = nil +} + +// Has returns true if the response chain has a response +func (r *ResponseChain) Has() bool { + return r.resp != nil +} + +// Request is request of current response +func (r *ResponseChain) Request() *http.Request { + if r.resp == nil { + return nil + } + return r.resp.Request +} + +// Response is response of current response +func (r *ResponseChain) Response() *http.Response { + return r.resp +} + +// reset without releasing the buffers +// useful for redirect chain +func (r *ResponseChain) reset() { + r.headers.Reset() + r.body.Reset() + r.fullResponse.Reset() +} diff --git a/pkg/protocols/http/httputils/internal.go b/pkg/protocols/http/httputils/internal.go new file mode 100644 index 0000000000..98f261328e --- /dev/null +++ b/pkg/protocols/http/httputils/internal.go @@ -0,0 +1,47 @@ +package httputils + +import ( + "bytes" + "errors" + "io" + "net/http" + "strings" +) + +// implementations copied from stdlib + +// errNoBody is a sentinel error value used by failureToReadBody so we +// can detect that the lack of body was intentional. +var errNoBody = errors.New("sentinel error value") + +// failureToReadBody is an io.ReadCloser that just returns errNoBody on +// Read. It's swapped in when we don't actually want to consume +// the body, but need a non-nil one, and want to distinguish the +// error from reading the dummy body. +type failureToReadBody struct{} + +func (failureToReadBody) Read([]byte) (int, error) { return 0, errNoBody } +func (failureToReadBody) Close() error { return nil } + +// emptyBody is an instance of empty reader. +var emptyBody = io.NopCloser(strings.NewReader("")) + +// drainBody reads all of b to memory and then returns two equivalent +// ReadClosers yielding the same bytes. +// +// It returns an error if the initial slurp of all bytes fails. It does not attempt +// to make the returned ReadClosers have identical error-matching behavior. +func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) { + if b == nil || b == http.NoBody { + // No copying needed. Preserve the magic sentinel meaning of NoBody. + return http.NoBody, http.NoBody, nil + } + var buf bytes.Buffer + if _, err = buf.ReadFrom(b); err != nil { + return nil, b, err + } + if err = b.Close(); err != nil { + return nil, b, err + } + return io.NopCloser(&buf), io.NopCloser(bytes.NewReader(buf.Bytes())), nil +} diff --git a/pkg/protocols/http/httputils/misc.go b/pkg/protocols/http/httputils/misc.go new file mode 100644 index 0000000000..6405e512e4 --- /dev/null +++ b/pkg/protocols/http/httputils/misc.go @@ -0,0 +1,21 @@ +package httputils + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v3/pkg/types" + mapsutil "github.com/projectdiscovery/utils/maps" +) + +// if template contains more than 1 request and matchers require requestcondition from +// both requests , then we need to request for event from interactsh even if current request +// doesnot use interactsh url in it +func GetInteractshURLSFromEvent(event map[string]interface{}) []string { + interactshUrls := map[string]struct{}{} + for k, v := range event { + if strings.HasPrefix(k, "interactsh-url") { + interactshUrls[types.ToString(v)] = struct{}{} + } + } + return mapsutil.GetKeys(interactshUrls) +} diff --git a/pkg/protocols/http/httputils/normalization.go b/pkg/protocols/http/httputils/normalization.go new file mode 100644 index 0000000000..010e7aa212 --- /dev/null +++ b/pkg/protocols/http/httputils/normalization.go @@ -0,0 +1,77 @@ +package httputils + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "io" + "net/http" + "strings" + + "github.com/pkg/errors" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + + stringsutil "github.com/projectdiscovery/utils/strings" +) + +// readNNormalizeRespBody performs normalization on the http response object. +// and fills body buffer with actual response body. +func readNNormalizeRespBody(rc *ResponseChain, body *bytes.Buffer) (err error) { + response := rc.resp + // net/http doesn't automatically decompress the response body if an + // encoding has been specified by the user in the request so in case we have to + // manually do it. + + origBody := rc.resp.Body + // wrap with decode if applicable + wrapped, err := wrapDecodeReader(response) + if err != nil { + wrapped = origBody + } + // read response body to buffer + _, err = body.ReadFrom(wrapped) + if err != nil { + if strings.Contains(err.Error(), "gzip: invalid header") { + // its invalid gzip but we will still use it from original body + _, err = body.ReadFrom(origBody) + if err != nil { + return errors.Wrap(err, "could not read response body after gzip error") + } + } + if stringsutil.ContainsAny(err.Error(), "unexpected EOF", "read: connection reset by peer", "user canceled") { + // keep partial body and continue (skip error) (add meta header in response for debugging) + response.Header.Set("x-nuclei-ignore-error", err.Error()) + return nil + } + return errors.Wrap(err, "could not read response body") + } + return nil +} + +// wrapDecodeReader wraps a decompression reader around the response body if it's compressed +// using gzip or deflate. +func wrapDecodeReader(resp *http.Response) (rc io.ReadCloser, err error) { + switch resp.Header.Get("Content-Encoding") { + case "gzip": + rc, err = gzip.NewReader(resp.Body) + case "deflate": + rc, err = zlib.NewReader(resp.Body) + default: + rc = resp.Body + } + if err != nil { + return nil, err + } + // handle GBK encoding + if isContentTypeGbk(resp.Header.Get("Content-Type")) { + rc = io.NopCloser(transform.NewReader(rc, simplifiedchinese.GBK.NewDecoder())) + } + return rc, nil +} + +// isContentTypeGbk checks if the content-type header is gbk +func isContentTypeGbk(contentType string) bool { + contentType = strings.ToLower(contentType) + return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") +} diff --git a/pkg/protocols/http/httputils/response.go b/pkg/protocols/http/httputils/response.go new file mode 100644 index 0000000000..5803e1271d --- /dev/null +++ b/pkg/protocols/http/httputils/response.go @@ -0,0 +1,52 @@ +package httputils + +import ( + "bytes" + "fmt" + "io" + "net/http" + + protocolutil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" +) + +// DumpResponseIntoBuffer dumps a http response without allocating a new buffer +// for the response body. +func DumpResponseIntoBuffer(resp *http.Response, body bool, buff *bytes.Buffer) (err error) { + if resp == nil { + return fmt.Errorf("response is nil") + } + save := resp.Body + savecl := resp.ContentLength + + if !body { + // For content length of zero. Make sure the body is an empty + // reader, instead of returning error through failureToReadBody{}. + if resp.ContentLength == 0 { + resp.Body = emptyBody + } else { + resp.Body = failureToReadBody{} + } + } else if resp.Body == nil { + resp.Body = emptyBody + } else { + save, resp.Body, err = drainBody(resp.Body) + if err != nil { + return err + } + } + err = resp.Write(buff) + if err == errNoBody { + err = nil + } + resp.Body = save + resp.ContentLength = savecl + return +} + +// DrainResponseBody drains the response body and closes it. +func DrainResponseBody(resp *http.Response) { + defer resp.Body.Close() + // don't reuse connection and just close if body length is more than 2 * MaxBodyRead + // to avoid DOS + _, _ = io.CopyN(io.Discard, resp.Body, 2*protocolutil.MaxBodyRead) +} diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 660b2430e9..ac022d242e 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "net/http/httputil" "strconv" "strings" "sync" @@ -32,12 +31,14 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" protocolutil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/rawhttp" + convUtil "github.com/projectdiscovery/utils/conversion" "github.com/projectdiscovery/utils/reader" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" @@ -400,7 +401,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa MatchFunc: request.Match, ExtractFunc: request.Extract, } - allOASTUrls := getInteractshURLsFromEvent(event.InternalEvent) + allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent) allOASTUrls = append(allOASTUrls, generatedHttpRequest.interactshURLs...) request.options.Interactsh.RequestEvent(sliceutil.Dedupe(allOASTUrls), requestData) gotMatches = request.options.Interactsh.AlreadyMatched(requestData) @@ -685,12 +686,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ callback(event) return err } - defer func() { - if resp.StatusCode != http.StatusSwitchingProtocols { - _, _ = io.CopyN(io.Discard, resp.Body, drainReqSize) - } - resp.Body.Close() - }() var curlCommand string if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil && !request.Race { @@ -706,55 +701,39 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err) duration := time.Since(timeStart) - - dumpedResponseHeaders, err := httputil.DumpResponse(resp, false) - if err != nil { - return errors.Wrap(err, "could not dump http response") - } - - var dumpedResponse []redirectedResponse - var gotData []byte - // If the status code is HTTP 101, we should not proceed with reading body. - if resp.StatusCode != http.StatusSwitchingProtocols { - var bodyReader io.Reader - if request.MaxSize != 0 { - bodyReader = io.LimitReader(resp.Body, int64(request.MaxSize)) - } else if request.options.Options.ResponseReadSize != 0 { - bodyReader = io.LimitReader(resp.Body, int64(request.options.Options.ResponseReadSize)) - } else { - bodyReader = resp.Body - } - data, err := io.ReadAll(bodyReader) - if err != nil { - // Ignore body read due to server misconfiguration errors - if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { - gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error()) - } else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error - return errors.Wrap(err, "could not read http body") - } - } - gotData = data - resp.Body.Close() - - dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) - if err != nil { - return errors.Wrap(err, "could not read http response with redirect chain") - } - } else { - dumpedResponse = []redirectedResponse{{resp: resp, fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} - } - - // if nuclei-project is enabled store the response if not previously done - if request.options.ProjectFile != nil && !fromCache { - if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil { - return errors.Wrap(err, "could not store in project file") - } - } - - for _, response := range dumpedResponse { - if response.resp == nil { - continue // Skip nil responses - } + // define max body read limit + maxBodylimit := -1 // stick to default 4MB + if request.MaxSize > 0 { + maxBodylimit = request.MaxSize + } else if request.options.Options.ResponseReadSize != 0 { + maxBodylimit = request.options.Options.ResponseReadSize + } + + // respChain is http response chain that reads response body + // efficiently by reusing buffers and does all decoding and optimizations + respChain := httputils.NewResponseChain(resp, int64(maxBodylimit)) + defer respChain.Close() // reuse buffers + + // we only intend to log/save the final redirected response + // i.e why we have to use sync.Once to ensure it's only done once + var errx error + onceFunc := sync.OnceFunc(func() { + // if nuclei-project is enabled store the response if not previously done + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, respChain.Body().Bytes()); err != nil { + errx = errors.Wrap(err, "could not store in project file") + } + } + }) + + // evaluate responses continiously until first redirect request in reverse order + for respChain.Has() { + // fill buffers, read response body and reuse connection + if err := respChain.Fill(); err != nil { + return errors.Wrap(err, "could not generate response chain") + } + // save response to projectfile + onceFunc() matchedURL := input.MetaInput.Input if generatedRequest.rawRequest != nil { if generatedRequest.rawRequest.FullURL != "" { @@ -767,14 +746,14 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ matchedURL = generatedRequest.request.URL.String() } // Give precedence to the final URL from response - if response.resp.Request != nil { - if responseURL := response.resp.Request.URL.String(); responseURL != "" { + if respChain.Request() != nil { + if responseURL := respChain.Request().URL.String(); responseURL != "" { matchedURL = responseURL } } finalEvent := make(output.InternalEvent) - outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + outputEvent := request.responseToDSLMap(respChain.Response(), input.MetaInput.Input, matchedURL, convUtil.String(dumpedRequest), respChain.FullResponse().String(), respChain.Body().String(), respChain.Headers().String(), duration, generatedRequest.meta) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) if request.options.HasTemplateCtx(input.MetaInput) { @@ -819,9 +798,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ event.UsesInteractsh = true } - responseContentType := resp.Header.Get("Content-Type") - isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize - dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input) + responseContentType := respChain.Response().Header.Get("Content-Type") + isResponseTruncated := request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize + dumpResponse(event, request, respChain.FullResponse().Bytes(), formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input) callback(event) @@ -829,8 +808,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() { return nil } + // proceed with previous response + // we evaluate operators recursively for each response + // until we reach the first redirect response + if !respChain.Previous() { + break + } } - return nil + // return project file save error if any + return errx } // handleSignature of the http request diff --git a/pkg/protocols/http/utils.go b/pkg/protocols/http/utils.go index d6b43547bb..fe893ebe60 100644 --- a/pkg/protocols/http/utils.go +++ b/pkg/protocols/http/utils.go @@ -1,119 +1,13 @@ package http import ( - "bytes" - "compress/gzip" - "compress/zlib" "io" - "net/http" - "net/http/httputil" "strings" - "github.com/pkg/errors" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" - protoUtil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" - "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/rawhttp" - mapsutil "github.com/projectdiscovery/utils/maps" - stringsutil "github.com/projectdiscovery/utils/strings" ) -type redirectedResponse struct { - headers []byte - body []byte - fullResponse []byte - resp *http.Response -} - -// dumpResponseWithRedirectChain dumps a http response with the -// complete http redirect chain. -// -// It preserves the order in which responses were given to requests -// and returns the data to the user for matching and viewing in that order. -// -// Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923 -func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) { - var response []redirectedResponse - - respData, err := httputil.DumpResponse(resp, false) - if err != nil { - return nil, err - } - respObj := redirectedResponse{ - headers: respData, - body: body, - resp: resp, - fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), - } - if err := normalizeResponseBody(resp, &respObj); err != nil { - return nil, err - } - response = append(response, respObj) - - var redirectResp *http.Response - if resp != nil && resp.Request != nil { - redirectResp = resp.Request.Response - } - for redirectResp != nil { - var body []byte - - respData, err := httputil.DumpResponse(redirectResp, false) - if err != nil { - break - } - if redirectResp.Body != nil { - body, _ = protoUtil.LimitBodyRead(redirectResp.Body) - } - respObj := redirectedResponse{ - headers: respData, - body: body, - resp: redirectResp, - fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), - } - if err := normalizeResponseBody(redirectResp, &respObj); err != nil { - return nil, err - } - response = append(response, respObj) - redirectResp = redirectResp.Request.Response - } - return response, nil -} - -// normalizeResponseBody performs normalization on the http response object. -func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error { - var err error - // net/http doesn't automatically decompress the response body if an - // encoding has been specified by the user in the request so in case we have to - // manually do it. - dataOrig := response.body - response.body, err = handleDecompression(resp, response.body) - // in case of error use original data - if err != nil { - response.body = dataOrig - } - response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body) - - // Decode gbk response content-types - // gb18030 supersedes gb2312 - responseContentType := resp.Header.Get("Content-Type") - if isContentTypeGbk(responseContentType) { - response.fullResponse, err = decodeGBK(response.fullResponse) - if err != nil { - return errors.Wrap(err, "could not gbk decode") - } - - // the uncompressed body needs to be decoded to standard utf8 - response.body, err = decodeGBK(response.body) - if err != nil { - return errors.Wrap(err, "could not gbk decode") - } - } - return nil -} - // dump creates a dump of the http request in form of a byte slice func dump(req *generatedRequest, reqURL string) ([]byte, error) { if req.request != nil { @@ -122,60 +16,3 @@ func dump(req *generatedRequest, reqURL string) ([]byte, error) { rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes} return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions) } - -// handleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically) -func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte, err error) { - if resp == nil { - return bodyOrig, nil - } - - var reader io.ReadCloser - switch resp.Header.Get("Content-Encoding") { - case "gzip": - reader, err = gzip.NewReader(bytes.NewReader(bodyOrig)) - case "deflate": - reader, err = zlib.NewReader(bytes.NewReader(bodyOrig)) - default: - return bodyOrig, nil - } - if err != nil { - return nil, err - } - defer reader.Close() - - bodyDec, err = io.ReadAll(reader) - if err != nil { - return bodyOrig, err - } - return bodyDec, nil -} - -// decodeGBK converts GBK to UTF-8 -func decodeGBK(s []byte) ([]byte, error) { - I := bytes.NewReader(s) - O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) - d, e := io.ReadAll(O) - if e != nil { - return nil, e - } - return d, nil -} - -// isContentTypeGbk checks if the content-type header is gbk -func isContentTypeGbk(contentType string) bool { - contentType = strings.ToLower(contentType) - return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") -} - -// if template contains more than 1 request and matchers require requestcondition from -// both requests , then we need to request for event from interactsh even if current request -// doesnot use interactsh url in it -func getInteractshURLsFromEvent(event map[string]interface{}) []string { - interactshUrls := map[string]struct{}{} - for k, v := range event { - if strings.HasPrefix(k, "interactsh-url") { - interactshUrls[types.ToString(v)] = struct{}{} - } - } - return mapsutil.GetKeys(interactshUrls) -} diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 54752d3607..b1275a0d9d 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -91,6 +91,10 @@ type Request struct { // cache any variables that may be needed for operation. options *protocols.ExecutorOptions `yaml:"-" json:"-"` + + preConditionCompiled *goja.Program `yaml:"-" json:"-"` + + scriptCompiled *goja.Program `yaml:"-" json:"-"` } // Compile compiles the request generators preparing any requests possible. @@ -196,13 +200,21 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { }, }) } + opts.Cleanup = func(runtime *goja.Runtime) { + _ = runtime.GlobalObject().Delete("set") + _ = runtime.GlobalObject().Delete("updatePayload") + } args := compiler.NewExecuteArgs() allVars := generators.MergeMaps(options.Variables.GetAll(), options.Options.Vars.AsMap(), request.options.Constants) // proceed with whatever args we have args.Args, _ = request.evaluateArgs(allVars, options, true) - result, err := request.options.JsCompiler.ExecuteWithOptions(request.Init, args, opts) + initCompiled, err := goja.Compile("", request.Init, false) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not compile init code: %s", err) + } + result, err := request.options.JsCompiler.ExecuteWithOptions(initCompiled, args, opts) if err != nil { return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) } @@ -217,6 +229,24 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } } + // compile pre-condition if any + if request.PreCondition != "" { + preConditionCompiled, err := goja.Compile("", request.PreCondition, false) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not compile pre-condition: %s", err) + } + request.preConditionCompiled = preConditionCompiled + } + + // compile actual source code + if request.Code != "" { + scriptCompiled, err := goja.Compile("", request.Code, false) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not compile javascript code: %s", err) + } + request.scriptCompiled = scriptCompiled + } + return nil } @@ -307,7 +337,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV } argsCopy.TemplateCtx = templateCtx.GetAll() - result, err := request.options.JsCompiler.ExecuteWithOptions(request.PreCondition, argsCopy, &compiler.ExecuteOptions{Timeout: request.Timeout}) + result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy, &compiler.ExecuteOptions{Timeout: request.Timeout}) if err != nil { return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) } @@ -425,18 +455,21 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte argsCopy.TemplateCtx = map[string]interface{}{} } - var requestData = []byte(request.Code) var interactshURLs []string if request.options.Interactsh != nil { - var transformedData string - transformedData, interactshURLs = request.options.Interactsh.Replace(string(request.Code), []string{}) - requestData = []byte(transformedData) + if argsCopy.Args != nil { + for k, v := range argsCopy.Args { + var urls []string + v, urls = request.options.Interactsh.Replace(fmt.Sprint(v), []string{}) + if len(urls) > 0 { + interactshURLs = append(interactshURLs, urls...) + argsCopy.Args[k] = v + } + } + } } - results, err := request.options.JsCompiler.ExecuteWithOptions(string(requestData), argsCopy, &compiler.ExecuteOptions{ - Pool: false, - Timeout: request.Timeout, - }) + results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy, &compiler.ExecuteOptions{Timeout: request.Timeout}) if err != nil { // shouldn't fail even if it returned error instead create a failure event results = compiler.ExecuteResult{"success": false, "error": err.Error()} diff --git a/pkg/protocols/utils/reader.go b/pkg/protocols/utils/reader.go index 62ad53cd0c..e6ed52530b 100644 --- a/pkg/protocols/utils/reader.go +++ b/pkg/protocols/utils/reader.go @@ -18,11 +18,21 @@ type LimitResponseBody struct { // NewLimitResponseBody wraps response body with a limit reader. // thus only allowing MaxBodyRead bytes to be read. i.e 4MB func NewLimitResponseBody(body io.ReadCloser) io.ReadCloser { + return NewLimitResponseBodyWithSize(body, MaxBodyRead) +} + +// NewLimitResponseBody wraps response body with a limit reader. +// thus only allowing MaxBodyRead bytes to be read. i.e 4MB +func NewLimitResponseBodyWithSize(body io.ReadCloser, size int64) io.ReadCloser { if body == nil { return nil } + if size == -1 { + // stick to default 4MB + size = MaxBodyRead + } return &LimitResponseBody{ - Reader: io.LimitReader(body, MaxBodyRead), + Reader: io.LimitReader(body, size), Closer: body, } } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index c5e378396a..ca9b54bf5f 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -6,11 +6,11 @@ import ( "strings" "sync/atomic" + "github.com/dop251/goja" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" @@ -24,6 +24,7 @@ type TemplateExecuter struct { options *protocols.ExecutorOptions engine TemplateEngine results *atomic.Bool + program *goja.Program } // Both executer & Executor are correct spellings (its open to interpretation) @@ -47,11 +48,11 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut // we use a dummy input here because goal of flow executor at this point is to just check // syntax and other things are correct before proceeding to actual execution // during execution new instance of flow will be created as it is tightly coupled with lot of executor options - var err error - e.engine, err = flow.NewFlowExecutor(requests, scan.NewScanContext(contextargs.NewWithInput("dummy")), options, e.results) + p, err := goja.Compile("flow.js", options.Flow, false) if err != nil { - return nil, fmt.Errorf("could not create flow executor: %s", err) + return nil, fmt.Errorf("could not compile flow: %s", err) } + e.program = p } else { // Review: // multiproto engine is only used if there is more than one protocol in template @@ -84,6 +85,10 @@ func (e *TemplateExecuter) Compile() error { return err } } + if e.engine == nil && e.options.Flow != "" { + // this is true for flow executor + return nil + } return e.engine.Compile() } @@ -158,7 +163,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // so in compile step earlier we compile it to validate javascript syntax and other things // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { - flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, results) + flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, results, e.program) if err != nil { ctx.LogError(err) return false, fmt.Errorf("could not create flow executor: %s", err) diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 0cae628fbd..33821ac005 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -11,7 +11,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -40,12 +39,11 @@ type FlowExecutor struct { options *protocols.ExecutorOptions // javascript runtime reference and compiled program - jsVM *goja.Runtime program *goja.Program // compiled js program // protocol requests and their callback functions allProtocols map[string][]protocols.Request - protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js + protoFunctions map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js // logic related variables results *atomic.Bool @@ -58,7 +56,7 @@ type FlowExecutor struct { // NewFlowExecutor creates a new flow executor from a list of requests // Note: Unlike other engine for every target x template flow needs to be compiled and executed everytime // unlike other engines where we compile once and execute multiple times -func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, options *protocols.ExecutorOptions, results *atomic.Bool) (*FlowExecutor, error) { +func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, options *protocols.ExecutorOptions, results *atomic.Bool, program *goja.Program) (*FlowExecutor, error) { allprotos := make(map[string][]protocols.Request) for _, req := range requests { switch req.Type() { @@ -96,10 +94,10 @@ func NewFlowExecutor(requests []protocols.Request, ctx *scan.ScanContext, option ReadOnly: atomic.Bool{}, Map: make(map[string]error), }, - protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, + protoFunctions: map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value{}, results: results, - jsVM: protocolstate.NewJSRuntime(), ctx: ctx, + program: program, } return f, nil } @@ -130,7 +128,7 @@ func (f *FlowExecutor) Compile() error { f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Merge(allVars) // merge all variables into template context // ---- define callback functions/objects---- - f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} + f.protoFunctions = map[string]func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value{} // iterate over all protocols and generate callback functions for each protocol for p, requests := range f.allProtocols { // for each protocol build a requestMap with reqID and protocol request @@ -150,7 +148,7 @@ func (f *FlowExecutor) Compile() error { } // ---define hook that allows protocol/request execution from js----- // --- this is the actual callback that is executed when function is invoked in js---- - f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { + f.protoFunctions[proto] = func(call goja.FunctionCall, runtime *goja.Runtime) goja.Value { opts := &ProtoOptions{ protoName: proto, } @@ -169,10 +167,10 @@ func (f *FlowExecutor) Compile() error { } } } - return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) + return runtime.ToValue(f.requestExecutor(runtime, reqMap, opts)) } } - return f.registerBuiltInFunctions() + return nil } // ExecuteWithResults executes the flow and returns results @@ -192,11 +190,50 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(key, value) }) } + + // get a new runtime from pool + runtime := GetJSRuntime(f.options.Options) + defer PutJSRuntime(runtime) // put runtime back to pool + defer func() { + // remove set builtin + _ = runtime.GlobalObject().Delete("set") + _ = runtime.GlobalObject().Delete("template") + for proto := range f.protoFunctions { + _ = runtime.GlobalObject().Delete(proto) + } + + }() + defer func() { + if r := recover(); r != nil { + f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r)) + } + }() + if ctx.OnResult == nil { return fmt.Errorf("output callback cannot be nil") } + // before running register set of builtins + if err := runtime.Set("set", func(call goja.FunctionCall) goja.Value { + varName := call.Argument(0).Export() + varValue := call.Argument(1).Export() + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(types.ToString(varName), varValue) + return goja.Null() + }); err != nil { + return err + } + // also register functions that allow executing protocols from js + for proto, fn := range f.protoFunctions { + if err := runtime.Set(proto, fn); err != nil { + return err + } + } + // register template object + if err := runtime.Set("template", f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()); err != nil { + return err + } + // pass flow and execute the js vm and handle errors - _, err := f.jsVM.RunProgram(f.program) + _, err := runtime.RunProgram(f.program) if err != nil { ctx.LogError(err) return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go index 2032a74443..b7f589d375 100644 --- a/pkg/tmplexec/flow/flow_internal.go +++ b/pkg/tmplexec/flow/flow_internal.go @@ -2,24 +2,18 @@ package flow import ( "fmt" - "reflect" "sync/atomic" "github.com/dop251/goja" - "github.com/logrusorgru/aurora" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" - "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" - "github.com/projectdiscovery/nuclei/v3/pkg/types" mapsutil "github.com/projectdiscovery/utils/maps" ) // contains all internal/unexported methods of flow // requestExecutor executes a protocol/request and returns true if any matcher was found -func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { +func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { defer func() { // evaluate all variables after execution of each protocol variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()) @@ -27,7 +21,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req // to avoid polling update template variables everytime we execute a protocol var m map[string]interface{} = f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll() - _ = f.jsVM.Set("template", m) + _ = runtime.Set("template", m) }() matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool // if no id is passed execute all requests in sequence @@ -117,99 +111,3 @@ func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStat } } } - -// registerBuiltInFunctions registers all built in functions for the flow -func (f *FlowExecutor) registerBuiltInFunctions() error { - // currently we register following builtin functions - // log -> log to stdout with [JS] prefix should only be used for debugging - // set -> set a variable in template context - // proto(arg ...String) <- this is generic syntax of how a protocol/request binding looks in js - // we only register only those protocols that are available in template - - // we also register a map datatype called template with all template variables - // template -> all template variables are available in js template object - - if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { - // TODO: verify string interpolation and handle multiple args - arg := call.Argument(0).Export() - switch value := arg.(type) { - case string: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) - case map[string]interface{}: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) - default: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) - } - return call.Argument(0) // return the same value - }); err != nil { - return err - } - - if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { - varName := call.Argument(0).Export() - varValue := call.Argument(1).Export() - f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(types.ToString(varName), varValue) - return goja.Null() - }); err != nil { - return err - } - - // iterate provides global iterator function by handling null values or strings - if err := f.jsVM.Set("iterate", func(call goja.FunctionCall) goja.Value { - allVars := []any{} - for _, v := range call.Arguments { - if v.Export() == nil { - continue - } - if v.ExportType().Kind() == reflect.Slice { - // convert []datatype to []interface{} - // since it cannot be type asserted to []interface{} directly - rfValue := reflect.ValueOf(v.Export()) - for i := 0; i < rfValue.Len(); i++ { - allVars = append(allVars, rfValue.Index(i).Interface()) - } - } else { - allVars = append(allVars, v.Export()) - } - } - return f.jsVM.ToValue(allVars) - }); err != nil { - return err - } - - // add a builtin dedupe object - if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { - d := builtin.NewDedupe(f.jsVM) - obj := call.This - // register these methods - _ = obj.Set("Add", d.Add) - _ = obj.Set("Values", d.Values) - return nil - }); err != nil { - return err - } - - var m = f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll() - if m == nil { - m = map[string]interface{}{} - } - - if err := f.jsVM.Set("template", m); err != nil { - // all template variables are available in js template object - return err - } - - // register all protocols - for name, fn := range f.protoFunctions { - if err := f.jsVM.Set(name, fn); err != nil { - return err - } - } - - program, err := goja.Compile("flow", f.options.Flow, false) - if err != nil { - return err - } - f.program = program - return nil -} diff --git a/pkg/tmplexec/flow/vm.go b/pkg/tmplexec/flow/vm.go new file mode 100644 index 0000000000..41d6c39255 --- /dev/null +++ b/pkg/tmplexec/flow/vm.go @@ -0,0 +1,94 @@ +package flow + +import ( + "reflect" + "sync" + + "github.com/dop251/goja" + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" + "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" + "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/remeh/sizedwaitgroup" +) + +type jsWaitGroup struct { + sync.Once + sg sizedwaitgroup.SizedWaitGroup +} + +var jsPool = &jsWaitGroup{} + +// GetJSRuntime returns a new JS runtime from pool +func GetJSRuntime(opts *types.Options) *goja.Runtime { + jsPool.Do(func() { + if opts.JsConcurrency < 100 { + opts.JsConcurrency = 100 + } + jsPool.sg = sizedwaitgroup.New(opts.JsConcurrency) + }) + jsPool.sg.Add() + return gojapool.Get().(*goja.Runtime) +} + +// PutJSRuntime returns a JS runtime to pool +func PutJSRuntime(runtime *goja.Runtime) { + defer jsPool.sg.Done() + gojapool.Put(runtime) +} + +// js runtime pool using sync.Pool +var gojapool = &sync.Pool{ + New: func() interface{} { + runtime := protocolstate.NewJSRuntime() + registerBuiltins(runtime) + return runtime + }, +} + +func registerBuiltins(runtime *goja.Runtime) { + _ = runtime.Set("log", func(call goja.FunctionCall) goja.Value { + // TODO: verify string interpolation and handle multiple args + arg := call.Argument(0).Export() + switch value := arg.(type) { + case string: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + case map[string]interface{}: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) + default: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + } + return call.Argument(0) // return the same value + }) + + _ = runtime.Set("iterate", func(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) + } + } + return runtime.ToValue(allVars) + }) + + _ = runtime.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { + d := builtin.NewDedupe(runtime) + obj := call.This + // register these methods + _ = obj.Set("Add", d.Add) + _ = obj.Set("Values", d.Values) + return nil + }) +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 9e57b118f7..5397eac3e2 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -366,6 +366,8 @@ type Options struct { EnableCloudUpload bool // ScanID is the scan ID to use for cloud upload ScanID string + // JsConcurrency is the number of concurrent js routines to run + JsConcurrency int } // ShouldLoadResume resume file