Skip to content

Commit

Permalink
add additional json fields: port,ip,scheme,url (#4417)
Browse files Browse the repository at this point in the history 
* add additional json fields: port,ip,scheme,url

* include host field in case of ip input
  • Loading branch information
@tarunKoyalwar
tarunKoyalwar authored Nov 28, 2023
1 parent ce5df9c commit 6e969cb
Show file tree
Hide file tree
Showing 10 changed files with 183 additions and 19 deletions.
24 changes: 22 additions & 2 deletions pkg/output/output.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
fileutil "github.com/projectdiscovery/utils/file"
Expand Down Expand Up @@ -62,7 +63,7 @@ type StandardWriter struct {
severityColors func(severity.Severity) string
storeResponse bool
storeResponseDir string
omitTemplate bool
omitTemplate bool
DisableStdout bool
AddNewLinesOutputFile bool // by default this is only done for stdout
}
Expand Down Expand Up @@ -132,6 +133,12 @@ type ResultEvent struct {
Type string `json:"type"`
// Host is the host input on which match was found.
Host string `json:"host,omitempty"`
// Port is port of the host input on which match was found (if applicable).
Port string `json:"port,omitempty"`
// Scheme is the scheme of the host input on which match was found (if applicable).
Scheme string `json:"scheme,omitempty"`
// URL is the Base URL of the host input on which match was found (if applicable).
URL string `json:"url,omitempty"`
// Path is the path input on which match was found.
Path string `json:"path,omitempty"`
// Matched contains the matched input in its transformed form.
Expand Down Expand Up @@ -347,14 +354,27 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
if event["template-info"] != nil {
templateInfo = event["template-info"].(model.Info)
}
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(event["host"]))
if types.ToString(event["ip"]) != "" {
fields.Ip = types.ToString(event["ip"])
}
if types.ToString(event["path"]) != "" {
fields.Path = types.ToString(event["path"])
}

data := &ResultEvent{
Template: templatePath,
TemplateURL: templateURL,
TemplateID: types.ToString(event["template-id"]),
TemplatePath: types.ToString(event["template-path"]),
Info: templateInfo,
Type: types.ToString(event["type"]),
Host: types.ToString(event["host"]),
Host: fields.Host,
Path: fields.Path,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Request: types.ToString(event["request"]),
Response: types.ToString(event["response"]),
MatcherStatus: false,
Expand Down
9 changes: 9 additions & 0 deletions pkg/protocols/code/code.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,12 +244,21 @@ func (request *Request) Type() templateTypes.ProtocolType {
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["input"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
Type: types.ToString(wrapped.InternalEvent["type"]),
Matched: types.ToString(wrapped.InternalEvent["input"]),
Host: fields.Host,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
Expand Down
16 changes: 14 additions & 2 deletions pkg/protocols/headless/operators.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)

Expand Down Expand Up @@ -125,17 +126,28 @@ func (request *Request) GetCompiledOperators() []*operators.Operators {
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
if types.ToString(wrapped.InternalEvent["path"]) != "" {
fields.Path = types.ToString(wrapped.InternalEvent["path"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Path: fields.Path,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["data"]),
TemplateEncoded: request.options.EncodeTemplate(),
Expand Down
15 changes: 13 additions & 2 deletions pkg/protocols/http/operators.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,18 +149,29 @@ func (request *Request) GetCompiledOperators() []*operators.Operators {
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := utils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
if types.ToString(wrapped.InternalEvent["path"]) != "" {
fields.Path = types.ToString(wrapped.InternalEvent["path"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
Path: fields.Path,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: request.truncateResponse(wrapped.InternalEvent["response"]),
CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]),
Expand Down
10 changes: 8 additions & 2 deletions pkg/protocols/javascript/js.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -620,20 +620,26 @@ func (request *Request) getExcludePorts() string {
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["response"]),
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
TemplateEncoded: request.options.EncodeTemplate(),
Error: types.ToString(wrapped.InternalEvent["error"]),
}
Expand Down
19 changes: 13 additions & 6 deletions pkg/protocols/network/operators.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)

Expand Down Expand Up @@ -94,18 +95,24 @@ func (request *Request) GetCompiledOperators() []*operators.Operators {
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Port: fields.Port,
URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Metadata: wrapped.OperatorsResult.PayloadValues,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["data"]),
TemplateEncoded: request.options.EncodeTemplate(),
Expand Down
14 changes: 12 additions & 2 deletions pkg/protocols/ssl/ssl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,18 +390,28 @@ func (request *Request) Type() templateTypes.ProtocolType {
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
// in case scheme is not specified , we only connect to port 443 unless custom https port was specified
// like 8443 etc
if fields.Port == "80" {
fields.Port = "443"
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
TemplateEncoded: request.options.EncodeTemplate(),
Error: types.ToString(wrapped.InternalEvent["error"]),
}
Expand Down
71 changes: 71 additions & 0 deletions pkg/protocols/utils/fields.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package utils

import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
iputil "github.com/projectdiscovery/utils/ip"
urlutil "github.com/projectdiscovery/utils/url"
)

// JsonFields contains additional metadata fields for JSON output
type JsonFields struct {
Host string `json:"host,omitempty"`
Path string `json:"path,omitempty"`
Port string `json:"port,omitempty"`
Ip string `json:"ip,omitempty"`
Scheme string `json:"scheme,omitempty"`
URL string `json:"url,omitempty"`
}

// GetJsonFields returns the json fields for the request
func GetJsonFieldsFromURL(URL string) JsonFields {
parsed, err := urlutil.Parse(URL)
if err != nil {
return JsonFields{}
}
fields := JsonFields{
Port: parsed.Port(),
Scheme: parsed.Scheme,
URL: parsed.String(),
Path: parsed.Path,
}
if fields.Port == "" {
fields.Port = "80"
if fields.Scheme == "https" {
fields.Port = "443"
}
}
if iputil.IsIP(parsed.Host) {
fields.Ip = parsed.Host
}

fields.Host = parsed.Host
return fields
}

// GetJsonFieldsFromMetaInput returns the json fields for the request
func GetJsonFieldsFromMetaInput(ctx *contextargs.MetaInput) JsonFields {
input := ctx.Input
fields := JsonFields{
Ip: ctx.CustomIP,
}
parsed, err := urlutil.Parse(input)
if err != nil {
return fields
}
fields.Port = parsed.Port()
fields.Scheme = parsed.Scheme
fields.URL = parsed.String()
fields.Path = parsed.Path
if fields.Port == "" {
fields.Port = "80"
if fields.Scheme == "https" {
fields.Port = "443"
}
}
if iputil.IsIP(parsed.Host) {
fields.Ip = parsed.Host
}

fields.Host = parsed.Host
return fields
}
9 changes: 7 additions & 2 deletions pkg/protocols/websocket/websocket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,18 +395,23 @@ var RequestPartDefinitions = map[string]string{
}

func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["response"]),
TemplateEncoded: request.options.EncodeTemplate(),
Expand Down
15 changes: 14 additions & 1 deletion pkg/testutils/testutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
)
Expand Down Expand Up @@ -166,14 +167,26 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven
if ti, ok := event["template-info"].(model.Info); ok {
templateInfo = ti
}
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(event["host"]))
if types.ToString(event["ip"]) != "" {
fields.Ip = types.ToString(event["ip"])
}
if types.ToString(event["path"]) != "" {
fields.Path = types.ToString(event["path"])
}
data := &output.ResultEvent{
Template: templatePath,
TemplateURL: templateURL,
TemplateID: types.ToString(event["template-id"]),
TemplatePath: types.ToString(event["template-path"]),
Info: templateInfo,
Type: types.ToString(event["type"]),
Host: types.ToString(event["host"]),
Path: fields.Path,
Host: fields.Host,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Request: types.ToString(event["request"]),
Response: types.ToString(event["response"]),
MatcherStatus: false,
Expand Down

0 comments on commit 6e969cb

Please sign in to comment.