From 29be9fa21f3a069ee9c483075b68d5c806e0a998 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 24 Oct 2024 16:53:30 +0530 Subject: [PATCH 1/3] bump gozero + improve debug mode (code protocol) --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ pkg/protocols/code/code.go | 23 ++++++++++++++++++++++- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index e4a71aec7d..fc8b40277b 100644 --- a/go.mod +++ b/go.mod @@ -38,9 +38,9 @@ require ( github.com/weppos/publicsuffix-go v0.30.2 github.com/xanzy/go-gitlab v0.107.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.29.0 + golang.org/x/net v0.30.0 golang.org/x/oauth2 v0.22.0 - golang.org/x/text v0.18.0 + golang.org/x/text v0.19.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -87,7 +87,7 @@ require ( github.com/projectdiscovery/goflags v0.1.64 github.com/projectdiscovery/gologger v1.1.28 github.com/projectdiscovery/gostruct v0.0.2 - github.com/projectdiscovery/gozero v0.0.2 + github.com/projectdiscovery/gozero v0.0.3-0.20241024112203-3377de55fc68 github.com/projectdiscovery/httpx v1.6.9 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 @@ -107,7 +107,7 @@ require ( github.com/yassinebenaid/godump v0.10.0 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 go.mongodb.org/mongo-driver v1.17.0 - golang.org/x/term v0.24.0 + golang.org/x/term v0.25.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 ) @@ -313,10 +313,10 @@ require ( go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mod v0.17.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 2223302430..fb52eebd85 100644 --- a/go.sum +++ b/go.sum @@ -861,8 +861,8 @@ github.com/projectdiscovery/gologger v1.1.28 h1:3OYYnoEaHAYkPm2c2KgUifBDPy93JCQq github.com/projectdiscovery/gologger v1.1.28/go.mod h1:rB7xNi4tcx/HqpQu4QgpFIopIg8SNF/VBc6cTeQLqWQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= -github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= +github.com/projectdiscovery/gozero v0.0.3-0.20241024112203-3377de55fc68 h1:A4JTEx7pJub5NLw8mg9DhIRodHU6YTaJYUlU6CtMSAA= +github.com/projectdiscovery/gozero v0.0.3-0.20241024112203-3377de55fc68/go.mod h1:9fP0jHtP/SAxO2J2t6OH+u8ZjI8v95+GHXvlwhB0iV8= github.com/projectdiscovery/hmap v0.0.62 h1:Pb3omgGQkRHB2EBDhgXniUXVC8Jz2CBzpSMKZYRUKFs= github.com/projectdiscovery/hmap v0.0.62/go.mod h1:pNheW3ukrLOcRUDFWrq8OAZiizhosDonIKRVg/4PGnM= github.com/projectdiscovery/httpx v1.6.9 h1:ihyFclesLjvQpiJpRIlAYeebapyIbOI/arDAvvy1ES8= @@ -1212,8 +1212,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1306,8 +1306,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1408,8 +1408,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1422,8 +1422,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1440,8 +1440,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 4503b2117f..b3344d08d4 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -86,6 +86,12 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { Args: request.Args, EarlyCloseFileDescriptor: true, } + + if options.Options.Debug || options.Options.DebugResponse { + // enable debug mode for gozero + gozeroOptions.DebugMode = true + } + engine, err := gozero.New(gozeroOptions) if err != nil { return errorutil.NewWithErr(err).Msgf("[%s] engines '%s' not available on host", options.TemplateID, strings.Join(request.Engine, ",")) @@ -239,7 +245,22 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Debug().Msgf("[%s] Dumped Executed Source Code for %v\n\n%v\n", request.options.TemplateID, input.MetaInput.Input, interpretEnvVars(request.Source, allvars)) + gologger.Debug().MsgFunc(func() string { + dashes := strings.Repeat("-", 15) + sb := &strings.Builder{} + sb.WriteString(fmt.Sprintf("[%s] Dumped Executed Source Code for input/stdin: '%v'", request.options.TemplateID, input.MetaInput.Input)) + sb.WriteString(fmt.Sprintf("\n%v\n%v\n%v\n", dashes, "Source Code:", dashes)) + sb.WriteString(interpretEnvVars(request.Source, allvars)) + sb.WriteString("\n") + sb.WriteString(fmt.Sprintf("\n%v\n%v\n%v\n", dashes, "Command Executed:", dashes)) + sb.WriteString(interpretEnvVars(gOutput.Command, allvars)) + sb.WriteString("\n") + sb.WriteString(fmt.Sprintf("\n%v\n%v\n%v\n", dashes, "Command Output:", dashes)) + sb.WriteString(gOutput.DebugData.String()) + sb.WriteString("\n") + sb.WriteString("[WRN] Command Output here is stdout+sterr, in response variables they are seperate (use -v -svd flags for more details)") + return sb.String() + }) } dataOutputString := fmtStdout(gOutput.Stdout.String()) From 3e02f0cad1fb91a74077f2336e198931427be09d Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 24 Oct 2024 17:45:57 +0530 Subject: [PATCH 2/3] run failed integration again assuming they are flaky --- cmd/integration-test/integration-test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 84ec6790f8..00726de5d2 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -104,10 +104,18 @@ func main() { if len(failedTestTemplatePaths) > 0 { if githubAction { - debug = true - fmt.Println("::group::Failed integration tests in debug mode") - _ = runTests(failedTestTemplatePaths) + // run failed tests again assuming they are flaky + // if they fail as well only then we assume that there is an actual issue + fmt.Println("::group::Running failed tests again") + failedTestTemplatePaths = runTests(failedTestTemplatePaths) fmt.Println("::endgroup::") + + if len(failedTestTemplatePaths) > 0 { + debug = true + fmt.Println("::group::Failed integration tests in debug mode") + _ = runTests(failedTestTemplatePaths) + fmt.Println("::endgroup::") + } } os.Exit(1) From 3f9bd6aac607eeaa954ed1cb6c363b665a8e299e Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 25 Oct 2024 23:34:19 +0530 Subject: [PATCH 3/3] dep update --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fc8b40277b..5ed96a001e 100644 --- a/go.mod +++ b/go.mod @@ -87,7 +87,7 @@ require ( github.com/projectdiscovery/goflags v0.1.64 github.com/projectdiscovery/gologger v1.1.28 github.com/projectdiscovery/gostruct v0.0.2 - github.com/projectdiscovery/gozero v0.0.3-0.20241024112203-3377de55fc68 + github.com/projectdiscovery/gozero v0.0.3 github.com/projectdiscovery/httpx v1.6.9 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 diff --git a/go.sum b/go.sum index fb52eebd85..47500f2c5d 100644 --- a/go.sum +++ b/go.sum @@ -861,8 +861,8 @@ github.com/projectdiscovery/gologger v1.1.28 h1:3OYYnoEaHAYkPm2c2KgUifBDPy93JCQq github.com/projectdiscovery/gologger v1.1.28/go.mod h1:rB7xNi4tcx/HqpQu4QgpFIopIg8SNF/VBc6cTeQLqWQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/gozero v0.0.3-0.20241024112203-3377de55fc68 h1:A4JTEx7pJub5NLw8mg9DhIRodHU6YTaJYUlU6CtMSAA= -github.com/projectdiscovery/gozero v0.0.3-0.20241024112203-3377de55fc68/go.mod h1:9fP0jHtP/SAxO2J2t6OH+u8ZjI8v95+GHXvlwhB0iV8= +github.com/projectdiscovery/gozero v0.0.3 h1:tsYkrSvWw4WdIUJyisd4MB1vRiw1X57TuVVk3p8Z3G8= +github.com/projectdiscovery/gozero v0.0.3/go.mod h1:MpJ37Dsh94gy2EKqaemdeh+CzduGVB2SDfhr6Upsjew= github.com/projectdiscovery/hmap v0.0.62 h1:Pb3omgGQkRHB2EBDhgXniUXVC8Jz2CBzpSMKZYRUKFs= github.com/projectdiscovery/hmap v0.0.62/go.mod h1:pNheW3ukrLOcRUDFWrq8OAZiizhosDonIKRVg/4PGnM= github.com/projectdiscovery/httpx v1.6.9 h1:ihyFclesLjvQpiJpRIlAYeebapyIbOI/arDAvvy1ES8=