Merge branch 'dev' of https://github.com/projectdiscovery/nuclei into…

… nuclei-dast-server

go.mod

@@ -21,11 +21,11 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.1.1
 	github.com/projectdiscovery/fastdialer v0.2.13
-	github.com/projectdiscovery/hmap v0.0.69
+	github.com/projectdiscovery/hmap v0.0.70
 	github.com/projectdiscovery/interactsh v1.2.0
-	github.com/projectdiscovery/rawhttp v0.1.76
+	github.com/projectdiscovery/rawhttp v0.1.77
 	github.com/projectdiscovery/retryabledns v1.0.87
-	github.com/projectdiscovery/retryablehttp-go v1.0.88
+	github.com/projectdiscovery/retryablehttp-go v1.0.89
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -89,20 +89,20 @@ require (
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
 	github.com/projectdiscovery/goflags v0.1.65
-	github.com/projectdiscovery/gologger v1.1.33
+	github.com/projectdiscovery/gologger v1.1.34
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.3
 	github.com/projectdiscovery/httpx v1.6.9
 	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.64
+	github.com/projectdiscovery/ratelimit v0.0.65
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.8
 	github.com/projectdiscovery/uncover v1.0.9
 	github.com/projectdiscovery/useragent v0.0.78
-	github.com/projectdiscovery/utils v0.3.0
-	github.com/projectdiscovery/wappalyzergo v0.2.5
+	github.com/projectdiscovery/utils v0.4.1
+	github.com/projectdiscovery/wappalyzergo v0.2.6
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466

go.sum

@@ -864,8 +864,8 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7
 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
 github.com/projectdiscovery/dsl v0.3.3 h1:4Ij5S86cHlb6xFrS7+5zAiJPeBt5h970XBTHqeTkpyU=
 github.com/projectdiscovery/dsl v0.3.3/go.mod h1:DAjSeaogLM9f0Ves2zDc/vbJrfcv+kEmS51p0dLLaPI=
-github.com/projectdiscovery/fastdialer v0.2.12-0.20241205195710-bb4879dd1d39 h1:NfDFJnc0r33XDYJLvBjm7kV1pc6RhDhLco/W2j459Wo=
-github.com/projectdiscovery/fastdialer v0.2.12-0.20241205195710-bb4879dd1d39/go.mod h1:R1lMBMgp1orUO39tOe9kujDbEO2iQNQZgDM/2TqIRf8=
+github.com/projectdiscovery/fastdialer v0.2.11 h1:DTx2vJ3tytv34wDe+Oh72L7v9pZWhzNGFJgwheN0n1Q=
+github.com/projectdiscovery/fastdialer v0.2.11/go.mod h1:jjDMLl+hnKoSSP82eWPxn8U+KivlWqf/o3pSz4n4dik=
 github.com/projectdiscovery/fastdialer v0.2.13 h1:5XzSv0hwITzRAMwyoJ9GFZSTVtaI4jmwER968TbDLbI=
 github.com/projectdiscovery/fastdialer v0.2.13/go.mod h1:T1EaYHbWmCnVHSYz12nAjnHMNFEfGMLLw37cb0k1X3A=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -876,14 +876,14 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
 github.com/projectdiscovery/goflags v0.1.65 h1:rjoj+5lP/FDzgeM0WILUTX9AOOnw0J0LXtl8P1SVeGE=
 github.com/projectdiscovery/goflags v0.1.65/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g=
-github.com/projectdiscovery/gologger v1.1.33 h1:wQxaQ8p/0Rx89lowBp0PnY2QSWiqf9QW1vGYAllsVJ4=
-github.com/projectdiscovery/gologger v1.1.33/go.mod h1:P/WwqKstshQATJxN39V0KJ9ZuiGLOizmSqHIYrrz1T4=
+github.com/projectdiscovery/gologger v1.1.34 h1:/66ev5Rq+7zvve1ZAA1V3HhYCbMACqwjDHvd4DKW4jQ=
+github.com/projectdiscovery/gologger v1.1.34/go.mod h1:dQDwORPIM5FM31EO6n2/VO+x5MSzOUpWi0EbccKD7fk=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.3 h1:tsYkrSvWw4WdIUJyisd4MB1vRiw1X57TuVVk3p8Z3G8=
 github.com/projectdiscovery/gozero v0.0.3/go.mod h1:MpJ37Dsh94gy2EKqaemdeh+CzduGVB2SDfhr6Upsjew=
-github.com/projectdiscovery/hmap v0.0.69 h1:e30pCr6JShf/UyJmKQpx++Yceiijw4GWj3lFHGZ1yko=
-github.com/projectdiscovery/hmap v0.0.69/go.mod h1:LgZHcgcxOvA3X8tuFtfu4dofJjAHAfpMno27Jx0J34w=
+github.com/projectdiscovery/hmap v0.0.70 h1:1TtvmzJNntKbU9CJI7W5auchg62lzgFYaG4BljwrS1o=
+github.com/projectdiscovery/hmap v0.0.70/go.mod h1:sugZdHVusZTc45CKtZw7y460C7+JS5SY6ODwlE1i8tI=
 github.com/projectdiscovery/httpx v1.6.9 h1:ihyFclesLjvQpiJpRIlAYeebapyIbOI/arDAvvy1ES8=
 github.com/projectdiscovery/httpx v1.6.9/go.mod h1:zQtX5CtcDYXzIRWne1ztCVtqG0sXCnx84tFwfMHoB8Q=
 github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -898,16 +898,16 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
 github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
-github.com/projectdiscovery/ratelimit v0.0.64 h1:fDjCM+U9HqJU6+NBvJn0kY89PxD20iWvRFX1mLblRB8=
-github.com/projectdiscovery/ratelimit v0.0.64/go.mod h1:XVuisddIjhmdd9ukw+w90AkSJd3pEKBTG9fBvSjwVxs=
-github.com/projectdiscovery/rawhttp v0.1.76 h1:O2IoYSyG7unH5oW8r8j3539koCNkimyzcHFmCbx5BDU=
-github.com/projectdiscovery/rawhttp v0.1.76/go.mod h1:ZxvbdkRV2PBoCbJxHh9B0P0nC5gVG3p1Z5uiua3iC5I=
+github.com/projectdiscovery/ratelimit v0.0.65 h1:77TC2PCKjL/kODcEDxpse43WjVDpGaComMv/ae4FbiI=
+github.com/projectdiscovery/ratelimit v0.0.65/go.mod h1:7W9ep1g1C7+kLBq7CTJNliGJKVyRSlmdDdxQkxvnN4Q=
+github.com/projectdiscovery/rawhttp v0.1.77 h1:PYR/Eb+ijsQc/MgtwhsjFq0+5kBxg3m6D0HhNd3RV6c=
+github.com/projectdiscovery/rawhttp v0.1.77/go.mod h1:86TRSgWDEsbD3UWvcOS6lPVPzH0tk5ncBOaFQUrapYc=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.87 h1:MPEXVKdu89FEW23xIMpBzzvdegvtcAs7osSqHimBVOs=
 github.com/projectdiscovery/retryabledns v1.0.87/go.mod h1:snDTjRcmBj+iveber/o0jC0iLEkM6c0Sdo2IXe2O+fE=
-github.com/projectdiscovery/retryablehttp-go v1.0.88 h1:uR6T+i8Sy1isfG1KClhhsXnOqkOR6E8MAvuyOFq3T10=
-github.com/projectdiscovery/retryablehttp-go v1.0.88/go.mod h1:ktjiIKyej+plUeK9vksqRf3wGicqY3E1rW84V/O7p0M=
+github.com/projectdiscovery/retryablehttp-go v1.0.89 h1:JHzDdn4dRq7tCZbfL4kB6NUtau6EAt1OiEjS6mJsmO0=
+github.com/projectdiscovery/retryablehttp-go v1.0.89/go.mod h1:5WOMyK45vZlyHI7ZPeovQHpS0q5ldkvsQq4y1iaN81o=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -918,10 +918,10 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
 github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
 github.com/projectdiscovery/useragent v0.0.78 h1:YpgiY3qXpzygFA88SWVseAyWeV9ZKrIpDkfOY+mQ/UY=
 github.com/projectdiscovery/useragent v0.0.78/go.mod h1:SQgk2DZu1qCvYqBRYWs2sjenXqLEDnRw65wJJoolwZ4=
-github.com/projectdiscovery/utils v0.3.0 h1:JNRMUPMx2gSzdk3bOQPtXSucoz+qC1JVkibIn7+kPTk=
-github.com/projectdiscovery/utils v0.3.0/go.mod h1:k2XlmfaYO4k6T4vAyUa3Kn/0BxPTIlNiBFpM6nVCbz0=
-github.com/projectdiscovery/wappalyzergo v0.2.5 h1:DhPEgeD+9i6yg+aGlbkT4iOAspQfB5ZzrmpEhwoiMlA=
-github.com/projectdiscovery/wappalyzergo v0.2.5/go.mod h1:fXiqsyLHaX/ovBNUe/nX0318bWON6SHftLvgq1xaOq0=
+github.com/projectdiscovery/utils v0.4.1 h1:DRAqVMuLuo5NJChzXVdSqsKfAY3eFf9N1NX3Blhb0yU=
+github.com/projectdiscovery/utils v0.4.1/go.mod h1:tV//VyD+4qZYn3s3XCS7xDbhW20qjUjV6CWDDHE2VfQ=
+github.com/projectdiscovery/wappalyzergo v0.2.6 h1:lRNt/t7//79R1y749IO2jgR0DkIZ06jjpgWDQKoOXLI=
+github.com/projectdiscovery/wappalyzergo v0.2.6/go.mod h1:fXiqsyLHaX/ovBNUe/nX0318bWON6SHftLvgq1xaOq0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
 github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

pkg/fuzz/analyzers/time/analyzer.go

@@ -19,9 +19,10 @@ type Analyzer struct{}
 
 const (
 	DefaultSleepDuration             = int(7)
-	DefaultRequestsLimit             = int(3)
+	DefaultRequestsLimit             = int(4)
 	DefaultTimeCorrelationErrorRange = float64(0.15)
 	DefaultTimeSlopeErrorRange       = float64(0.30)
+	DefaultLowSleepTimeSeconds       = float64(3)
 
 	defaultSleepTimeDuration = 7 * time.Second
 )

pkg/fuzz/analyzers/time/time_delay.go

@@ -28,10 +28,16 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"strings"
 )
 
 type timeDelayRequestSender func(delay int) (float64, error)
 
+type requstsSentMetadata struct {
+	delay         int
+	delayReceived float64
+}
+
 // checkTimingDependency checks the timing dependency for a given request
 //
 // It alternates and sends first a high request, then a low request. Each time
@@ -50,38 +56,52 @@ func checkTimingDependency(
 	regression := newSimpleLinearRegression()
 	requestsLeft := requestsLimit
 
+	var requestsSent []requstsSentMetadata
 	for {
 		if requestsLeft <= 0 {
 			break
 		}
 
-		isCorrelationPossible, err := sendRequestAndTestConfidence(regression, highSleepTimeSeconds, requestSender)
+		isCorrelationPossible, delayRecieved, err := sendRequestAndTestConfidence(regression, highSleepTimeSeconds, requestSender)
 		if err != nil {
 			return false, "", err
 		}
 		if !isCorrelationPossible {
 			return false, "", nil
 		}
+		requestsSent = append(requestsSent, requstsSentMetadata{
+			delay:         highSleepTimeSeconds,
+			delayReceived: delayRecieved,
+		})
 
-		isCorrelationPossible, err = sendRequestAndTestConfidence(regression, 4, requestSender)
+		isCorrelationPossibleSecond, delayRecievedSecond, err := sendRequestAndTestConfidence(regression, int(DefaultLowSleepTimeSeconds), requestSender)
 		if err != nil {
 			return false, "", err
 		}
-		if !isCorrelationPossible {
+		if !isCorrelationPossibleSecond {
 			return false, "", nil
 		}
 		requestsLeft = requestsLeft - 2
+
+		requestsSent = append(requestsSent, requstsSentMetadata{
+			delay:         int(DefaultLowSleepTimeSeconds),
+			delayReceived: delayRecievedSecond,
+		})
 	}
 
 	result := regression.IsWithinConfidence(correlationErrorRange, 1.0, slopeErrorRange)
 	if result {
-		resultReason := fmt.Sprintf(
+		var resultReason strings.Builder
+		resultReason.WriteString(fmt.Sprintf(
 			"[time_delay] made %d requests successfully, with a regression slope of %.2f and correlation %.2f",
 			requestsLimit,
 			regression.slope,
 			regression.correlation,
-		)
-		return result, resultReason, nil
+		))
+		for _, request := range requestsSent {
+			resultReason.WriteString(fmt.Sprintf("\n - delay: %ds, delayReceived: %fs", request.delay, request.delayReceived))
+		}
+		return result, resultReason.String(), nil
 	}
 	return result, "", nil
 }
@@ -91,22 +111,22 @@ func sendRequestAndTestConfidence(
 	regression *simpleLinearRegression,
 	delay int,
 	requestSender timeDelayRequestSender,
-) (bool, error) {
+) (bool, float64, error) {
 	delayReceived, err := requestSender(delay)
 	if err != nil {
-		return false, err
+		return false, 0, err
 	}
 
 	if delayReceived < float64(delay) {
-		return false, nil
+		return false, 0, nil
 	}
 
 	regression.AddPoint(float64(delay), delayReceived)
 
 	if !regression.IsWithinConfidence(0.3, 1.0, 0.5) {
-		return false, nil
+		return false, delayReceived, nil
 	}
-	return true, nil
+	return true, delayReceived, nil
 }
 
 type simpleLinearRegression struct {
@@ -180,7 +200,6 @@ func (o *simpleLinearRegression) Predict(x float64) float64 {
 
 func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64) bool {
 	// For now, just check correlation as originally done:
-	// You might later reintroduce slope checks:
 	// return math.Abs(expectedSlope-o.slope) < slopeErrorRange && o.correlation > 1.0 - correlationErrorRange
 	if o.count < 2 {
 		return true

pkg/fuzz/analyzers/time/time_delay_test.go

@@ -201,23 +201,6 @@ func linearSender(baseline, slope, noiseAmplitude float64) func(int) (float64, e
 	}
 }
 
-// changingBaselineSender simulates a baseline that changes after half the requests are done.
-func changingBaselineSender(initialBaseline, newBaseline, slope, noiseAmplitude float64, switchAfter int, counter *int) func(int) (float64, error) {
-	return func(delay int) (float64, error) {
-		time.Sleep(10 * time.Millisecond)
-		base := initialBaseline
-		if *counter >= switchAfter {
-			base = newBaseline
-		}
-		*counter++
-		noise := 0.0
-		if noiseAmplitude > 0 {
-			noise = (rand.Float64()*2 - 1) * noiseAmplitude
-		}
-		return base + slope*float64(delay) + noise, nil
-	}
-}
-
 // negativeSlopeSender just for completeness - higher delay = less observed time
 func negativeSlopeSender(baseline float64) func(int) (float64, error) {
 	return func(delay int) (float64, error) {
@@ -226,9 +209,6 @@ func negativeSlopeSender(baseline float64) func(int) (float64, error) {
 	}
 }
 
-// We assume you have an imported checkTimingDependency function. Adjust imports as needed.
-// func checkTimingDependency(...) (bool, string, error) { ... }
-
 func TestPerfectLinearSlopeOne_NoNoise(t *testing.T) {
 	match, reason, err := checkTimingDependency(
 		10,  // requestsLimit
@@ -313,27 +293,6 @@ func TestNegativeSlopeScenario(t *testing.T) {
 	}
 }
 
-func TestChangingBaseline(t *testing.T) {
-	counter := 0
-	// baseline = 2s initially, then after 5 requests it changes to 5s.
-	// slope=1 means observed = baseline + requested_delay.
-	// Even with changing baseline, a strong correlation should still appear if slope is consistent.
-	match, reason, err := checkTimingDependency(
-		12,
-		5,
-		0.2,
-		0.5,
-		changingBaselineSender(2.0, 5.0, 1.0, 0.1, 6, &counter),
-	)
-	if err != nil {
-		t.Fatalf("Error: %v", err)
-	}
-	// Still should see a linear relationship overall (requests with delay=5 should be consistently ~delay more than delay=1).
-	if !match {
-		t.Fatalf("Expected a match despite baseline changes. Reason: %s", reason)
-	}
-}
-
 func TestLargeNumberOfRequests(t *testing.T) {
 	// 20 requests, slope=1.0, no noise. Should be very stable and produce a very high correlation.
 	match, reason, err := checkTimingDependency(
@@ -408,7 +367,7 @@ func TestAlternatingSequences(t *testing.T) {
 		t.Fatalf("Expected a match but got none. Reason: %s", reason)
 	}
 	// Verify alternating sequence of delays
-	expectedDelays := []float64{15, 1, 15, 1}
+	expectedDelays := []float64{15, 4, 15, 4}
 	if !reflect.DeepEqual(generatedDelays, expectedDelays) {
 		t.Fatalf("Expected delays %v but got %v", expectedDelays, generatedDelays)
 	}

pkg/protocols/utils/variables.go

@@ -56,7 +56,7 @@ const (
 	Sd
 )
 
-// GenerateVariables will create default variables with context args
+// GenerateVariablesWithContextArgs will create default variables with context args
 func GenerateVariablesWithContextArgs(input *contextargs.Context, trailingSlash bool) map[string]interface{} {
 	parsed, err := urlutil.Parse(input.MetaInput.Input)
 	if err != nil {

pkg/testutils/integration.go

@@ -94,7 +94,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
 	return parts, nil
 }
 
-// RunNucleiArgsAndGetResults returns result,and runtime errors
+// RunNucleiWithArgsAndGetResults returns result,and runtime errors
 func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) {
 	cmd := exec.Command("./nuclei", args...)
 	cmd.Env = append(cmd.Env, ExtraEnvVars...)

pkg/utils/http_probe.go

@@ -13,7 +13,7 @@ var (
 	HttpSchemes = []string{"https", "http"}
 )
 
-// probeURL probes the scheme for a URL. first HTTPS is tried
+// ProbeURL probes the scheme for a URL. first HTTPS is tried
 // and if any errors occur http is tried. If none succeeds, probing
 // is abandoned for such URLs.
 func ProbeURL(input string, httpxclient *httpx.HTTPX) string {