template preprocessor + multi request variables indexing bug fix (#4262)

* add randstr preprocessor to defaults

* fix indexing in http + preprocessor integration test

* add multi-request integration test

* skip test if asnmap is down

cmd/integration-test/http.go

@@ -10,6 +10,7 @@ import (
 	"net/http/httputil"
 	"os"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -80,6 +81,8 @@ var httpTestcases = []TestCaseInfo{
 	{Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}},
 	{Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}},
 	{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
+	{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
+	{Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}},
 }
 
 type httpInteractshRequest struct{}
@@ -1475,3 +1478,56 @@ func (h *httpInteractshRequestsWithMCAnd) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 1)
 }
+
+// integration test to check if preprocessor i.e {{randstr}}
+// is working correctly
+type httpPreprocessor struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpPreprocessor) Execute(filePath string) error {
+	router := httprouter.New()
+	re := regexp.MustCompile(`[A-Za-z0-9]{25,}`)
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		value := r.URL.RequestURI()
+		if re.MatchString(value) {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprint(w, "ok")
+		} else {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, "not ok")
+		}
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type httpMultiRequest struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpMultiRequest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/ping", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "ping")
+	})
+	router.GET("/pong", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "pong")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}

integration_tests/protocols/http/http-preprocessor.yaml

@@ -0,0 +1,17 @@
+id: http-preprocessor
+
+info:
+  name: Test Http Preprocessor
+  author: pdteam
+  severity: info
+
+requests:
+  - raw:
+      - |
+        GET /?test={{randstr}} HTTP/1.1
+        Host: {{Hostname}}
+
+    matchers:
+      - type: status
+        status:
+          - 200

integration_tests/protocols/http/multi-request.yaml

@@ -0,0 +1,26 @@
+id: http-multi-request
+
+info:
+  name: http multi request template
+  author: pdteam
+  severity: info
+  description: template with multiple http request with combined logic
+  reference: https://example-reference-link
+
+# requestURI is reflected back as response body here
+http:
+  - raw:
+      - |
+        GET /ping HTTP/1.1
+        Host: {{Hostname}}
+     
+      - |
+        GET /pong HTTP/1.1
+        Host: {{Hostname}}
+    
+    matchers:
+        - type: dsl
+          dsl:
+            - 'body_1 == "ping"'
+            - 'body_2 == "pong"'
+          condition: and

pkg/core/inputs/hybrid/hmap_test.go

@@ -180,6 +180,10 @@ func Test_expandASNInputValue(t *testing.T) {
 			got = append(got, metainput.Input)
 			return nil
 		})
+		if len(got) == 0 {
+			// asnmap server is down
+			t.SkipNow()
+		}
 		// read the expected IPs from the file
 		fileContent, err := os.ReadFile(tt.expectedOutputFile)
 		require.Nil(t, err, "could not read the expectedOutputFile file")

pkg/protocols/http/request.go

@@ -793,7 +793,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		if request.NeedsRequestCondition() {
 			for k, v := range outputEvent {
 				key := fmt.Sprintf("%s_%d", k, requestCount)
-				if previousEvent[key] != nil {
+				if previousEvent != nil {
 					previousEvent[key] = v
 				}
 				finalEvent[key] = v

pkg/templates/compile.go

@@ -272,7 +272,7 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
 	// persist verified status value and then
 	// expand all preprocessor and reparse template
 
-	// === signature verification befoer preprocessors ===
+	// === signature verification before preprocessors ===
 	template, err := parseTemplate(data, options)
 	if err != nil {
 		return nil, err

pkg/templates/preprocessors.go

@@ -17,7 +17,9 @@ type Preprocessor interface {
 
 var (
 	preprocessorRegex    = regexp.MustCompile(`{{([a-z0-9_]+)}}`)
-	defaultPreprocessors = []Preprocessor{}
+	defaultPreprocessors = []Preprocessor{
+		&randStrPreprocessor{},
+	}
 )
 
 func getPreprocessors(preprocessor Preprocessor) []Preprocessor {