From c7284f6fe3b0008c624addab9a8d113740e2c0f3 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 19 Jul 2023 21:51:40 +0530 Subject: [PATCH 1/2] headless: automerge and other improvements --- v2/pkg/protocols/headless/engine/instance.go | 10 +- v2/pkg/protocols/headless/engine/page.go | 2 +- .../protocols/headless/engine/page_actions.go | 96 +++++++++++-------- v2/pkg/protocols/headless/request.go | 38 ++++++-- v2/pkg/protocols/http/build_request.go | 12 +-- .../{http/utils => utils/http}/requtils.go | 2 +- .../utils => utils/http}/requtils_test.go | 2 +- 7 files changed, 103 insertions(+), 59 deletions(-) rename v2/pkg/protocols/{http/utils => utils/http}/requtils.go (98%) rename v2/pkg/protocols/{http/utils => utils/http}/requtils_test.go (99%) diff --git a/v2/pkg/protocols/headless/engine/instance.go b/v2/pkg/protocols/headless/engine/instance.go index 90ffccac25..f0dae78280 100644 --- a/v2/pkg/protocols/headless/engine/instance.go +++ b/v2/pkg/protocols/headless/engine/instance.go @@ -17,6 +17,7 @@ type Instance struct { // redundant due to dependency cycle interactsh *interactsh.Client + requestLog map[string]string // contains actual request that was sent } // NewInstance creates a new instance for the current browser. @@ -35,7 +36,14 @@ func (b *Browser) NewInstance() (*Instance, error) { // We use a custom sleeper that sleeps from 100ms to 500 ms waiting // for an interaction. Used throughout rod for clicking, etc. browser = browser.Sleeper(func() utils.Sleeper { return maxBackoffSleeper(10) }) - return &Instance{browser: b, engine: browser}, nil + return &Instance{browser: b, engine: browser, requestLog: map[string]string{}}, nil +} + +// returns a map of [template-defined-urls] -> [actual-request-sent] +// Note: this does not include CORS or other requests while rendering that were not explicitly +// specified in template +func (i *Instance) GetRequestLog() map[string]string { + return i.requestLog } // Close closes all the tabs and pages for a browser instance diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index e23d2eb253..a566ab37a3 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -134,7 +134,7 @@ func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads m } } - data, err := createdPage.ExecuteActions(input, actions) + data, err := createdPage.ExecuteActions(input, actions, payloads) if err != nil { return nil, nil, err } diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index b96d4f264d..93cd5018c1 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -2,11 +2,8 @@ package engine import ( "context" - "net" - "net/url" "os" "path/filepath" - "regexp" "strconv" "strings" "sync" @@ -19,17 +16,21 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" "github.com/segmentio/ksuid" ) var ( errinvalidArguments = errors.New("invalid arguments provided") - reUrlWithPort = regexp.MustCompile(`{{BaseURL}}:(\d+)`) ) const ( @@ -39,17 +40,13 @@ const ( ) // ExecuteActions executes a list of actions on a page. -func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (map[string]string, error) { - baseURL, err := url.Parse(input.MetaInput.Input) - if err != nil { - return nil, err - } - +func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (map[string]string, error) { outData := make(map[string]string) + var err error for _, act := range actions { switch act.ActionType.ActionType { case ActionNavigate: - err = p.NavigateURL(act, outData, baseURL) + err = p.NavigateURL(act, outData, variables) case ActionScript: err = p.RunScript(act, outData) case ActionClick: @@ -237,25 +234,57 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { } // NavigateURL executes an ActionLoadURL actions loading a URL for the page. -func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error { - URL := p.getActionArgWithDefaultValues(action, "url") - if URL == "" { +func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[string]interface{}) error { + // input <- is input url from cli + // target <- is the url from template (ex: {{BaseURL}}/test) + input, err := urlutil.Parse(p.input.MetaInput.Input) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not parse url %s", p.input.MetaInput.Input) + } + target := p.getActionArgWithDefaultValues(action, "url") + if target == "" { return errinvalidArguments } - // Handle the dynamic value substitution here. - URL, parsed = baseURLWithTemplatePrefs(URL, parsed) - if strings.HasSuffix(parsed.Path, "/") && strings.Contains(URL, "{{BaseURL}}/") { - parsed.Path = strings.TrimSuffix(parsed.Path, "/") + // if target contains port ex: {{BaseURL}}:8080 use port specified in input + input, target = httputil.UpdateURLPortFromPayload(input, target) + hasTrailingSlash := httputil.HasTrailingSlash(target) + + // create vars from input url + defaultReqVars := protocolutils.GenerateVariables(input, hasTrailingSlash, contextargs.GenerateVariables(p.input)) + // merge all variables + // Note: ideally we should evaluate all available variables with reqvars + // but due to cyclic dependency between packages `engine` and `protocols` + // allvars are evaluated,merged and passed from headless package itself + // TODO: remove cyclic dependency between packages `engine` and `protocols` + allvars = generators.MergeMaps(allvars, defaultReqVars) + + if vardump.EnableVarDump { + gologger.Debug().Msgf("Final Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) } - parsedString := parsed.String() - final := replaceWithValues(URL, map[string]interface{}{ - "Hostname": parsed.Hostname(), - "BaseURL": parsedString, - }) - if err := p.page.Navigate(final); err != nil { - return errors.Wrap(err, "could not navigate") + // Evaluate the target url with all variables + target, err = expressions.Evaluate(target, allvars) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not evaluate url %s", target) + } + + reqURL, err := urlutil.ParseURL(target, true) + if err != nil { + return errorutil.NewWithTag("http", "failed to parse url %v while creating http request", target) + } + + // ===== parameter automerge ===== + // while merging parameters first preference is given to target params + finalparams := input.Params.Clone() + finalparams.Merge(reqURL.Params.Encode()) + reqURL.Params = finalparams + + // log all navigated requests + p.instance.requestLog[action.GetArg("url")] = reqURL.String() + + if err := p.page.Navigate(reqURL.String()); err != nil { + return errorutil.NewWithErr(err).Msgf("could not navigate to url %s", reqURL.String()) } return nil } @@ -609,23 +638,6 @@ func selectorBy(selector string) rod.SelectorType { } } -// baseURLWithTemplatePrefs returns the url for BaseURL keeping -// the template port and path preference over the user provided one. -func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { - // template port preference over input URL port if template has a port - matches := reUrlWithPort.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return data, parsed - } - port := matches[0][1] - parsed.Host = net.JoinHostPort(parsed.Hostname(), port) - data = strings.ReplaceAll(data, ":"+port, "") - if parsed.Path == "" { - parsed.Path = "/" - } - return data, parsed -} - func (p *Page) getActionArg(action *Action, arg string) string { return p.getActionArgWithValues(action, arg, nil) } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index b7f40eb415..8f77856f54 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -1,6 +1,7 @@ package headless import ( + "fmt" "net/url" "strings" "time" @@ -119,21 +120,31 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p } defer page.Close() + reqLog := instance.GetRequestLog() + navigatedURL := request.getLastNaviationURLWithLog(reqLog) // also known as matchedURL if there is a match + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil) request.options.Progress.IncrementRequests() - gologger.Verbose().Msgf("Sent Headless request to %s", input.MetaInput.Input) + gologger.Verbose().Msgf("Sent Headless request to %s", navigatedURL) reqBuilder := &strings.Builder{} if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input.MetaInput.Input) + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, navigatedURL) for _, act := range request.Steps { - actStepStr := act.String() - actStepStr = strings.ReplaceAll(actStepStr, "{{BaseURL}}", input.MetaInput.Input) - reqBuilder.WriteString("\t" + actStepStr + "\n") + if act.ActionType.ActionType == engine.ActionNavigate { + value := act.GetArg("url") + if reqLog[value] != "" { + reqBuilder.WriteString(fmt.Sprintf("\tnavigate => %v\n", reqLog[value])) + } else { + reqBuilder.WriteString(fmt.Sprintf("%v not found in %v\n", value, reqLog)) + } + } else { + actStepStr := act.String() + reqBuilder.WriteString("\t" + actStepStr + "\n") + } } gologger.Debug().Msgf(reqBuilder.String()) - } var responseBody string @@ -142,7 +153,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory()) + outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory()) for k, v := range out { outputEvent[k] = v } @@ -215,3 +226,16 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads } return nil } + +// getLastNaviationURL returns last successfully navigated URL +func (request *Request) getLastNaviationURLWithLog(reqLog map[string]string) string { + for i := len(request.Steps) - 1; i >= 0; i-- { + if request.Steps[i].ActionType.ActionType == engine.ActionNavigate { + templateURL := request.Steps[i].GetArg("url") + if reqLog[templateURL] != "" { + return reqLog[templateURL] + } + } + } + return "" +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index eecd9eadf4..af551d09cc 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -17,8 +17,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/utils" protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + httputil "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils/http" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" @@ -97,8 +97,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, hasTrailingSlash := false if !isRawRequest { // if path contains port ex: {{BaseURL}}:8080 use port specified in reqData - parsed, reqData = utils.UpdateURLPortFromPayload(parsed, reqData) - hasTrailingSlash = utils.HasTrailingSlash(reqData) + parsed, reqData = httputil.UpdateURLPortFromPayload(parsed, reqData) + hasTrailingSlash = httputil.HasTrailingSlash(reqData) } // defaultreqvars are vars generated from request/input ex: {{baseURL}}, {{Host}} etc @@ -362,13 +362,13 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st req.Body = bodyReader } if !r.request.Unsafe { - utils.SetHeader(req, "User-Agent", uarand.GetRandom()) + httputil.SetHeader(req, "User-Agent", uarand.GetRandom()) } // Only set these headers on non-raw requests if len(r.request.Raw) == 0 && !r.request.Unsafe { - utils.SetHeader(req, "Accept", "*/*") - utils.SetHeader(req, "Accept-Language", "en") + httputil.SetHeader(req, "Accept", "*/*") + httputil.SetHeader(req, "Accept-Language", "en") } if !LeaveDefaultPorts { diff --git a/v2/pkg/protocols/http/utils/requtils.go b/v2/pkg/protocols/utils/http/requtils.go similarity index 98% rename from v2/pkg/protocols/http/utils/requtils.go rename to v2/pkg/protocols/utils/http/requtils.go index 7be1f22591..97bb49f6e6 100644 --- a/v2/pkg/protocols/http/utils/requtils.go +++ b/v2/pkg/protocols/utils/http/requtils.go @@ -1,4 +1,4 @@ -package utils +package httputil import ( "regexp" diff --git a/v2/pkg/protocols/http/utils/requtils_test.go b/v2/pkg/protocols/utils/http/requtils_test.go similarity index 99% rename from v2/pkg/protocols/http/utils/requtils_test.go rename to v2/pkg/protocols/utils/http/requtils_test.go index 635d628556..056b13b81f 100644 --- a/v2/pkg/protocols/http/utils/requtils_test.go +++ b/v2/pkg/protocols/utils/http/requtils_test.go @@ -1,4 +1,4 @@ -package utils +package httputil import ( "testing" From ac8c929dc54fcfb0ff8a3a94b142e067a50d560c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 20 Jul 2023 18:30:31 +0530 Subject: [PATCH 2/2] fix typo in function signature --- v2/pkg/protocols/headless/request.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 8f77856f54..f9ce8e6a96 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -121,7 +121,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p defer page.Close() reqLog := instance.GetRequestLog() - navigatedURL := request.getLastNaviationURLWithLog(reqLog) // also known as matchedURL if there is a match + navigatedURL := request.getLastNavigationURLWithLog(reqLog) // also known as matchedURL if there is a match request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil) request.options.Progress.IncrementRequests() @@ -228,7 +228,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads } // getLastNaviationURL returns last successfully navigated URL -func (request *Request) getLastNaviationURLWithLog(reqLog map[string]string) string { +func (request *Request) getLastNavigationURLWithLog(reqLog map[string]string) string { for i := len(request.Steps) - 1; i >= 0; i-- { if request.Steps[i].ActionType.ActionType == engine.ActionNavigate { templateURL := request.Steps[i].GetArg("url")