Merge pull request #3573 from projectdiscovery/dev

nuclei v2.9.2

.github/workflows/build-test.yml

@@ -42,6 +42,7 @@ jobs:
         working-directory: v2/
 
       - name: Integration Tests
+        timeout-minutes: 50
         env:
           GH_ACTION: true
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

Dockerfile

@@ -1,5 +1,5 @@
 # Build
-FROM golang:1.20.2-alpine AS build-env
+FROM golang:1.20.3-alpine AS build-env
 RUN apk add build-base
 WORKDIR /app
 COPY . /app
@@ -8,7 +8,7 @@ RUN go mod download
 RUN go build ./cmd/nuclei
 
 # Release
-FROM alpine:3.17.2
+FROM alpine:3.17.3
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools chromium ca-certificates
 COPY --from=build-env /app/v2/nuclei /usr/local/bin/

README.md

@@ -161,7 +161,8 @@ OUTPUT:
    -ms, -matcher-status          display match failure status
    -me, -markdown-export string  directory to export results in markdown format
    -se, -sarif-export string     file to export results in SARIF format
-   -je, -json-export string      file to export results in JSON format as a JSON array. This can be memory intensive in larger scans.
+   -je, -json-export string      file to export results in JSON format as a JSON array. This can be memory intensive in larger scans
+   -jle, -jsonl-export string    file to export results in JSONL(ine) format as a list of line-delimited JSON objects
 
 CONFIGURATIONS:
    -config string                 path to the nuclei configuration file
@@ -264,7 +265,7 @@ DEBUG:
    -hc, -health-check        run diagnostic check up
 
 UPDATE:
-   -un, -update                      update nuclei engine to the latest released version
+   -up, -update                      update nuclei engine to the latest released version
    -ut, -update-templates            update nuclei-templates to latest released version
    -ud, -update-template-dir string  custom directory to install / update nuclei-templates
    -duc, -disable-update-check       disable automatic nuclei/templates update check

README_ID.md

@@ -140,7 +140,8 @@ OUTPUT:
    -ms, -matcher-status          display match failure status
    -me, -markdown-export string  directory to export results in markdown format
    -se, -sarif-export string     file to export results in SARIF format
-   -je, -json-export string      file to export results in JSON format as a JSON array. This can be memory intensive in larger scans.
+   -je, -json-export string      file to export results in JSON format as a JSON array. This can be memory intensive in larger scans
+   -jle, -jsonl-export string    file to export results in JSONL(ine) format as a list of line-delimited JSON objects
 
 CONFIGURATIONS:
    -config string                 path to the nuclei configuration file

integration_tests/http/interactsh-stop-at-first-match.yaml

@@ -8,9 +8,15 @@ info:
 requests:
   - method: GET
     path:
-      - "{{BaseURL}}"
-      - "{{BaseURL}}"
-      - "{{BaseURL}}"
+      - "{{BaseURL}}/?a=1"
+      - "{{BaseURL}}/?a=2"
+      - "{{BaseURL}}/?a=3"
+      - "{{BaseURL}}/?a=4"
+      - "{{BaseURL}}/?a=5"
+      - "{{BaseURL}}/?a=6"
+      - "{{BaseURL}}/?a=7"
+      - "{{BaseURL}}/?a=8"
+      - "{{BaseURL}}/?a=9"
     headers:
       url: 'http://{{interactsh-url}}'
 

integration_tests/http/variables.yaml

@@ -7,7 +7,7 @@ info:
 
 variables:
   a1: "value"
-  a2: "{{base64('hello')}}"
+  a2: "{{base64('{{Host}}')}}"
 
 requests:
   - raw:
@@ -16,11 +16,16 @@ requests:
         Host: {{FQDN}}
         Test: {{a1}}
         Another: {{a2}}
-
+        Email: {{ username }}
+    payloads:
+      username:
+        - jon.doe@{{ FQDN }}
     stop-at-first-match: true
     matchers-condition: or
     matchers:
       - type: word
+        condition: and
         words: 
           - "value"
-          - "aGVsbG8="
+          - "MTI3LjAuMC4x" # 127.0.0.1
+          - "[email protected]"

v2/cmd/cve-annotate/main.go

@@ -5,10 +5,13 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
 	"net/url"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -56,10 +59,7 @@ func process() error {
 	}
 	defer os.RemoveAll(tempDir)
 
-	client, err := nvd.NewClient(tempDir)
-	if err != nil {
-		return err
-	}
+	client := nvd.NewClientV2()
 	catalog := disk.NewCatalog(*templateDir)
 
 	paths, err := catalog.GetTemplatePath(*input)
@@ -109,7 +109,7 @@ var badRefs = []string{
 	"secunia.com/",
 }
 
-func getCVEData(client *nvd.Client, filePath, data string) {
+func getCVEData(client *nvd.ClientV2, filePath, data string) {
 	matches := idRegex.FindAllStringSubmatch(data, 1)
 	if len(matches) == 0 {
 		return
@@ -142,13 +142,18 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 		return
 	}
 	var cweID []string
-	for _, problemData := range cveItem.CVE.Problemtype.ProblemtypeData {
-		for _, description := range problemData.Description {
+	for _, weaknessData := range cveItem.Cve.Weaknesses {
+		for _, description := range weaknessData.Description {
 			cweID = append(cweID, description.Value)
 		}
 	}
-	cvssScore := cveItem.Impact.BaseMetricV3.CvssV3.BaseScore
-	cvssMetrics := cveItem.Impact.BaseMetricV3.CvssV3.VectorString
+	cvssData, err := getPrimaryCVSSData(cveItem)
+	if err != nil {
+		log.Printf("Could not get CVSS data %s: %s\n", cveName, err)
+		return
+	}
+	cvssScore := cvssData.BaseScore
+	cvssMetrics := cvssData.VectorString
 
 	// Perform some hacky string replacement to place the metadata in templates
 	infoBlockIndexData := data[strings.Index(data, "info:"):]
@@ -191,12 +196,13 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 		}
 	}
 	// If there is no description field, fill the description from CVE information
-	hasDescriptionData := len(cveItem.CVE.Description.DescriptionData) > 0
+	enDescription, err := getEnglishLangString(cveItem.Cve.Descriptions)
+	hasDescriptionData := err != nil
 	isDescriptionEmpty := infoBlock.Info.Description == ""
 	if isDescriptionEmpty && hasDescriptionData {
 		changed = true
 		// removes all new lines
-		description := stringsutil.ReplaceAll(cveItem.CVE.Description.DescriptionData[0].Value, "", "\n", "\\", "'", "\t")
+		description := stringsutil.ReplaceAll(enDescription, "", "\n", "\\", "'", "\t")
 		description += "\n"
 		infoBlock.Info.Description = description
 	}
@@ -205,13 +211,13 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 	var referenceDataURLs []string
 
 	// skip sites that are no longer alive
-	for _, reference := range cveItem.CVE.References.ReferenceData {
+	for _, reference := range cveItem.Cve.References {
 		if stringsutil.ContainsAny(reference.URL, badRefs...) {
 			continue
 		}
 		referenceDataURLs = append(referenceDataURLs, reference.URL)
 	}
-	hasReferenceData := len(cveItem.CVE.References.ReferenceData) > 0
+	hasReferenceData := len(cveItem.Cve.References) > 0
 	areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs)
 	referencesCount := len(infoBlock.Info.Reference)
 	if hasReferenceData && !areCveReferencesContained {
@@ -226,6 +232,36 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 		infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference))
 	}
 
+	cpeSet := map[string]bool{}
+	for _, config := range cveItem.Cve.Configurations {
+		// Right now this covers only simple configurations. More complex configurations can have multiple CPEs
+		if len(config.Nodes) == 1 {
+			changed = true
+			node := config.Nodes[0]
+			for _, match := range node.CpeMatch {
+				cpeSet[extractVersionlessCpe((match.Criteria))] = true
+			}
+		}
+	}
+	uniqueCpes := make([]string, 0, len(cpeSet))
+	for k := range cpeSet {
+		uniqueCpes = append(uniqueCpes, k)
+	}
+	if len(uniqueCpes) == 1 {
+		infoBlock.Info.Classification.Cpe = uniqueCpes[0]
+	}
+
+	epss, err := fetchEpss(cveName)
+	if err != nil {
+		log.Printf("Could not fetch Epss score: %s\n", err)
+		return
+	}
+	hasEpssChanged := epss != infoBlock.Info.Classification.EpssScore
+	if hasEpssChanged {
+		changed = true
+		infoBlock.Info.Classification.EpssScore = epss
+	}
+
 	var newInfoBlock bytes.Buffer
 	yamlEncoder := yaml.NewEncoder(&newInfoBlock)
 	yamlEncoder.SetIndent(yamlIndentSpaces)
@@ -243,6 +279,29 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 	}
 }
 
+func getPrimaryCVSSData(vuln nvd.Vulnerability) (nvd.CvssData, error) {
+	for _, data := range vuln.Cve.Metrics.CvssMetricV31 {
+		if data.Type == "Primary" {
+			return data.CvssData, nil
+		}
+	}
+	for _, data := range vuln.Cve.Metrics.CvssMetricV3 {
+		if data.Type == "Primary" {
+			return data.CvssData, nil
+		}
+	}
+	return nvd.CvssData{}, fmt.Errorf("no primary cvss metric found")
+}
+
+func getEnglishLangString(data []nvd.LangString) (string, error) {
+	for _, item := range data {
+		if item.Lang == "en" {
+			return item.Value, nil
+		}
+	}
+	return "", fmt.Errorf("no english item found")
+}
+
 func isSeverityMatchingCvssScore(severity string, score float64) string {
 	if score == 0.0 {
 		return ""
@@ -264,6 +323,51 @@ func isSeverityMatchingCvssScore(severity string, score float64) string {
 	return ""
 }
 
+func extractVersionlessCpe(cpe string) string {
+	parts := strings.Split(cpe, ":")
+	versionlessPart := parts[0:5]
+	rest := strings.Split(strings.Repeat("*", len(parts)-len(versionlessPart)), "")
+	return strings.Join(append(versionlessPart, rest...), ":")
+}
+
+type ApiFirstEpssResponse struct {
+	Status     string `json:"status"`
+	StatusCode int    `json:"status-code"`
+	Version    string `json:"version"`
+	Access     string `json:"access"`
+	Total      int    `json:"total"`
+	Offset     int    `json:"offset"`
+	Limit      int    `json:"limit"`
+	Data       []struct {
+		Cve        string `json:"cve"`
+		Epss       string `json:"epss"`
+		Percentile string `json:"percentile"`
+		Date       string `json:"date"`
+	} `json:"data"`
+}
+
+func fetchEpss(cveId string) (float64, error) {
+	resp, err := http.Get(fmt.Sprintf("https://api.first.org/data/v1/epss?cve=%s", cveId))
+	if err != nil {
+		return 0, fmt.Errorf("unable to fetch EPSS data from first.org: %v", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return 0, fmt.Errorf("unable to read reponse body: %v", err)
+	}
+	var parsedResp ApiFirstEpssResponse
+	err = json.Unmarshal(body, &parsedResp)
+	if err != nil {
+		return 0, fmt.Errorf("error while parsing EPSS response: %v", err)
+	}
+	if len(parsedResp.Data) != 1 {
+		return 0, fmt.Errorf("unexpected number of results in EPSS response. Expecting exactly 1, got %v", len(parsedResp.Data))
+	}
+	epss := parsedResp.Data[0].Epss
+	return strconv.ParseFloat(epss, 64)
+}
+
 type cisaKEVData struct {
 	Vulnerabilities []struct {
 		CVEID string `json:"cveID"`
@@ -392,6 +496,8 @@ type TemplateClassification struct {
 	CvssScore   float64 `yaml:"cvss-score,omitempty"`
 	CveId       string  `yaml:"cve-id,omitempty"`
 	CweId       string  `yaml:"cwe-id,omitempty"`
+	Cpe         string  `yaml:"cpe,omitempty"`
+	EpssScore   float64 `yaml:"epss-score,omitempty"`
 }
 
 type TemplateInfo struct {

v2/cmd/integration-test/code.go

@@ -88,7 +88,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
 	defaultOpts.Templates = goflags.StringSlice{templatePath}
 	defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
 
-	interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress)
+	interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
 	interactClient, err := interactsh.New(interactOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create interact client")
@@ -120,11 +120,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
 	}
 	executerOpts.WorkflowLoader = workflowLoader
 
-	configObject, err := config.ReadConfiguration()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not read configuration file")
-	}
-	store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts))
+	store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create loader")
 	}

v2/cmd/integration-test/fuzz.go

@@ -50,7 +50,7 @@ func (h *fuzzModeOverride) Execute(filePath string) error {
 	})
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", false, "-fuzzing-mode", "single", "-json")
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", debug, "-fuzzing-mode", "single", "-jsonl")
 	if err != nil {
 		return err
 	}
@@ -95,7 +95,7 @@ func (h *fuzzTypeOverride) Execute(filePath string) error {
 	})
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", false, "-fuzzing-type", "replace", "-json")
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", debug, "-fuzzing-type", "replace", "-jsonl")
 	if err != nil {
 		return err
 	}

v2/cmd/integration-test/http.go

@@ -47,8 +47,6 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/http-paths.yaml":                          &httpPaths{},
 	"http/request-condition.yaml":                   &httpRequestCondition{},
 	"http/request-condition-new.yaml":               &httpRequestCondition{},
-	"http/interactsh.yaml":                          &httpInteractshRequest{},
-	"http/interactsh-stop-at-first-match.yaml":      &httpInteractshStopAtFirstMatchRequest{},
 	"http/self-contained.yaml":                      &httpRequestSelfContained{},
 	"http/self-contained-file-input.yaml":           &httpRequestSelfContainedFileInput{},
 	"http/get-case-insensitive.yaml":                &httpGetCaseInsensitive{},
@@ -71,7 +69,6 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/get-without-scheme.yaml":                  &httpGetWithoutScheme{},
 	"http/cl-body-without-header.yaml":              &httpCLBodyWithoutHeader{},
 	"http/cl-body-with-header.yaml":                 &httpCLBodyWithHeader{},
-	"http/default-matcher-condition.yaml":           &httpDefaultMatcherCondition{},
 }
 
 type httpInteractshRequest struct{}
@@ -164,6 +161,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
+	// polling is asyncronous, so the interactions may be retrieved after the first request
 	return expectResultsCount(results, 1)
 }
 
@@ -1050,7 +1048,7 @@ type httpVariables struct{}
 func (h *httpVariables) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "%s\n%s", r.Header.Get("Test"), r.Header.Get("Another"))
+		fmt.Fprintf(w, "%s\n%s\n%s", r.Header.Get("Test"), r.Header.Get("Another"), r.Header.Get("Email"))
 	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()