feat: changed the logics a bit + misc changes and additions

go.mod

@@ -20,11 +20,11 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.1.1
-	github.com/projectdiscovery/fastdialer v0.2.10
+	github.com/projectdiscovery/fastdialer v0.2.13
 	github.com/projectdiscovery/hmap v0.0.69
 	github.com/projectdiscovery/interactsh v1.2.0
 	github.com/projectdiscovery/rawhttp v0.1.76
-	github.com/projectdiscovery/retryabledns v1.0.86
+	github.com/projectdiscovery/retryabledns v1.0.87
 	github.com/projectdiscovery/retryablehttp-go v1.0.88
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0

go.sum

@@ -864,8 +864,10 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7
 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
 github.com/projectdiscovery/dsl v0.3.3 h1:4Ij5S86cHlb6xFrS7+5zAiJPeBt5h970XBTHqeTkpyU=
 github.com/projectdiscovery/dsl v0.3.3/go.mod h1:DAjSeaogLM9f0Ves2zDc/vbJrfcv+kEmS51p0dLLaPI=
-github.com/projectdiscovery/fastdialer v0.2.10 h1:5iciZXMPdynbk/9iuqkJT1gqMXwzgEpFSWdoj/5CHCo=
-github.com/projectdiscovery/fastdialer v0.2.10/go.mod h1:21rwXMecVsPVdSvON8Up761/GgxC4OSc9Rvx5LNH5fY=
+github.com/projectdiscovery/fastdialer v0.2.12-0.20241205195710-bb4879dd1d39 h1:NfDFJnc0r33XDYJLvBjm7kV1pc6RhDhLco/W2j459Wo=
+github.com/projectdiscovery/fastdialer v0.2.12-0.20241205195710-bb4879dd1d39/go.mod h1:R1lMBMgp1orUO39tOe9kujDbEO2iQNQZgDM/2TqIRf8=
+github.com/projectdiscovery/fastdialer v0.2.13 h1:5XzSv0hwITzRAMwyoJ9GFZSTVtaI4jmwER968TbDLbI=
+github.com/projectdiscovery/fastdialer v0.2.13/go.mod h1:T1EaYHbWmCnVHSYz12nAjnHMNFEfGMLLw37cb0k1X3A=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -902,8 +904,8 @@ github.com/projectdiscovery/rawhttp v0.1.76 h1:O2IoYSyG7unH5oW8r8j3539koCNkimyzc
 github.com/projectdiscovery/rawhttp v0.1.76/go.mod h1:ZxvbdkRV2PBoCbJxHh9B0P0nC5gVG3p1Z5uiua3iC5I=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.86 h1:8YMJGJ94lFBKKN3t7NOzJfbGsZoh9qNpi49xdfJcZVc=
-github.com/projectdiscovery/retryabledns v1.0.86/go.mod h1:5PhXvlLkEFmlYOt9i4wiKA1eONLrNiZ6DQE88Ph9rgU=
+github.com/projectdiscovery/retryabledns v1.0.87 h1:MPEXVKdu89FEW23xIMpBzzvdegvtcAs7osSqHimBVOs=
+github.com/projectdiscovery/retryabledns v1.0.87/go.mod h1:snDTjRcmBj+iveber/o0jC0iLEkM6c0Sdo2IXe2O+fE=
 github.com/projectdiscovery/retryablehttp-go v1.0.88 h1:uR6T+i8Sy1isfG1KClhhsXnOqkOR6E8MAvuyOFq3T10=
 github.com/projectdiscovery/retryablehttp-go v1.0.88/go.mod h1:ktjiIKyej+plUeK9vksqRf3wGicqY3E1rW84V/O7p0M=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=

pkg/fuzz/analyzers/analyzers.go

@@ -81,18 +81,11 @@ func ApplyPayloadTransformations(value string) string {
 }
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-const (
-	letterIdxBits = 6                    // 6 bits to represent a letter index
-	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
-)
 
 func randStringBytesMask(n int) string {
 	b := make([]byte, n)
-	for i := 0; i < n; {
-		if idx := int(random.Int63() & letterIdxMask); idx < len(letterBytes) {
-			b[i] = letterBytes[idx]
-			i++
-		}
+	for i := range b {
+		b[i] = letterBytes[random.Intn(len(letterBytes))]
 	}
 	return string(b)
 }

pkg/fuzz/analyzers/time/analyzer.go

@@ -18,12 +18,12 @@ import (
 type Analyzer struct{}
 
 const (
-	DefaultSleepDuration             = int(5)
-	DefaultRequestsLimit             = int(4)
+	DefaultSleepDuration             = int(7)
+	DefaultRequestsLimit             = int(3)
 	DefaultTimeCorrelationErrorRange = float64(0.15)
 	DefaultTimeSlopeErrorRange       = float64(0.30)
 
-	defaultSleepTimeDuration = 5 * time.Second
+	defaultSleepTimeDuration = 7 * time.Second
 )
 
 var _ analyzers.Analyzer = &Analyzer{}
@@ -129,6 +129,7 @@ func (a *Analyzer) Analyze(options *analyzers.Options) (bool, string, error) {
 		}
 		return timeTaken, nil
 	}
+
 	matched, matchReason, err := checkTimingDependency(
 		requestsLimit,
 		sleepDuration,
@@ -147,14 +148,18 @@ func (a *Analyzer) Analyze(options *analyzers.Options) (bool, string, error) {
 
 // doHTTPRequestWithTimeTracing does a http request with time tracing
 func doHTTPRequestWithTimeTracing(req *retryablehttp.Request, httpclient *retryablehttp.Client) (float64, error) {
-	var ttfb time.Duration
-	var start time.Time
+	var serverTime time.Duration
+	var wroteRequest time.Time
 
 	trace := &httptrace.ClientTrace{
-		GotFirstResponseByte: func() { ttfb = time.Since(start) },
+		WroteHeaders: func() {
+			wroteRequest = time.Now()
+		},
+		GotFirstResponseByte: func() {
+			serverTime = time.Since(wroteRequest)
+		},
 	}
 	req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
-	start = time.Now()
 	resp, err := httpclient.Do(req)
 	if err != nil {
 		return 0, errors.Wrap(err, "could not do request")
@@ -164,5 +169,5 @@ func doHTTPRequestWithTimeTracing(req *retryablehttp.Request, httpclient *retrya
 	if err != nil {
 		return 0, errors.Wrap(err, "could not read response body")
 	}
-	return ttfb.Seconds(), nil
+	return serverTime.Seconds(), nil
 }

pkg/fuzz/analyzers/time/time_delay.go

@@ -6,6 +6,10 @@
 // Advantages of this approach are many compared to the old approach of
 // heuristics of sleep time.
 //
+// NOTE: This algorithm has been heavily modified after being introduced
+// in nuclei. Now the logic has sever bug fixes and improvements and
+// has been evolving to be more stable.
+//
 // As we are building a statistical model, we can predict if the delay
 // is random or not very quickly. Also, the payloads are alternated to send
 // a very high sleep and a very low sleep. This way the comparison is
@@ -59,7 +63,7 @@ func checkTimingDependency(
 			return false, "", nil
 		}
 
-		isCorrelationPossible, err = sendRequestAndTestConfidence(regression, 1, requestSender)
+		isCorrelationPossible, err = sendRequestAndTestConfidence(regression, 4, requestSender)
 		if err != nil {
 			return false, "", err
 		}
@@ -105,17 +109,14 @@ func sendRequestAndTestConfidence(
 	return true, nil
 }
 
-// simpleLinearRegression is a simple linear regression model that can be updated at runtime.
-// It is based on the same algorithm in ZAP for doing timing checks.
 type simpleLinearRegression struct {
-	count          float64
-	independentSum float64
-	dependentSum   float64
+	count float64
 
-	// Variances
-	independentVarianceN float64
-	dependentVarianceN   float64
-	sampleCovarianceN    float64
+	sumX  float64
+	sumY  float64
+	sumXX float64
+	sumYY float64
+	sumXY float64
 
 	slope       float64
 	intercept   float64
@@ -124,48 +125,65 @@ type simpleLinearRegression struct {
 
 func newSimpleLinearRegression() *simpleLinearRegression {
 	return &simpleLinearRegression{
-		slope:       1,
-		correlation: 1,
+		// Start everything at zero until we have data
+		slope:       0.0,
+		intercept:   0.0,
+		correlation: 0.0,
 	}
 }
 
 func (o *simpleLinearRegression) AddPoint(x, y float64) {
-	independentResidualAdjustment := x - o.independentSum/o.count
-	dependentResidualAdjustment := y - o.dependentSum/o.count
-
 	o.count += 1
-	o.independentSum += x
-	o.dependentSum += y
-
-	if math.IsNaN(independentResidualAdjustment) {
+	o.sumX += x
+	o.sumY += y
+	o.sumXX += x * x
+	o.sumYY += y * y
+	o.sumXY += x * y
+
+	// Need at least two points for meaningful calculation
+	if o.count < 2 {
 		return
 	}
 
-	independentResidual := x - o.independentSum/o.count
-	dependentResidual := y - o.dependentSum/o.count
-
-	o.independentVarianceN += independentResidual * independentResidualAdjustment
-	o.dependentVarianceN += dependentResidual * dependentResidualAdjustment
-	o.sampleCovarianceN += independentResidual * dependentResidualAdjustment
+	n := o.count
+	meanX := o.sumX / n
+	meanY := o.sumY / n
+
+	// Compute sample variances and covariance
+	varX := (o.sumXX - n*meanX*meanX) / (n - 1)
+	varY := (o.sumYY - n*meanY*meanY) / (n - 1)
+	covXY := (o.sumXY - n*meanX*meanY) / (n - 1)
+
+	// If varX is zero, slope cannot be computed meaningfully.
+	// This would mean all X are the same, so handle that edge case.
+	if varX == 0 {
+		o.slope = 0.0
+		o.intercept = meanY // Just the mean
+		o.correlation = 0.0 // No correlation since all X are identical
+		return
+	}
 
-	o.slope = o.sampleCovarianceN / o.independentVarianceN
-	o.correlation = o.slope * math.Sqrt(o.independentVarianceN/o.dependentVarianceN)
-	o.correlation *= o.correlation
+	o.slope = covXY / varX
+	o.intercept = meanY - o.slope*meanX
 
-	// NOTE: zap had the reverse formula, changed it to the correct one
-	// for intercept. Verify if this is correct.
-	o.intercept = o.dependentSum/o.count - o.slope*(o.independentSum/o.count)
-	if math.IsNaN(o.correlation) {
-		o.correlation = 1
+	// If varX or varY are zero, we cannot compute correlation properly.
+	if varX > 0 && varY > 0 {
+		o.correlation = covXY / (math.Sqrt(varX) * math.Sqrt(varY))
+	} else {
+		o.correlation = 0.0
 	}
 }
 
 func (o *simpleLinearRegression) Predict(x float64) float64 {
 	return o.slope*x + o.intercept
 }
 
-func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64,
-) bool {
+func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64) bool {
+	// For now, just check correlation as originally done:
+	// You might later reintroduce slope checks:
+	// return math.Abs(expectedSlope-o.slope) < slopeErrorRange && o.correlation > 1.0 - correlationErrorRange
+	if o.count < 2 {
+		return true
+	}
 	return o.correlation > 1.0-correlationErrorRange
-	//math.Abs(expectedSlope-o.slope) < slopeErrorRange
 }