Skip to content

Commit

Permalink
Merge branch 'dev' into feat-error-enhancements
Browse files Browse the repository at this point in the history
  • Loading branch information
ehsandeep authored May 14, 2024
2 parents 1818f8a + 4874729 commit 8e81d7d
Show file tree
Hide file tree
Showing 32 changed files with 409 additions and 49 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ TEMPLATES:
-nss, -no-strict-syntax disable strict syntax check on templates
-td, -template-display displays the templates content
-tl list all available templates
-tgl list all available tags
-sign signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
-code enable loading code protocol-based templates
-dut, -disable-unsigned-templates disable running unsigned templates or templates with mismatched signature
Expand Down
1 change: 1 addition & 0 deletions cmd/integration-test/integration-test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ var (
"dns": dnsTestCases,
"workflow": workflowTestcases,
"loader": loaderTestcases,
"profile-loader": profileLoaderTestcases,
"websocket": websocketTestCases,
"headless": headlessTestcases,
"whois": whoisTestCases,
Expand Down
54 changes: 54 additions & 0 deletions cmd/integration-test/profile-loader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package main

import (
"fmt"

"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
errorutil "github.com/projectdiscovery/utils/errors"
)

var profileLoaderTestcases = []TestCaseInfo{
{Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}},
{Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}},
{Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}},
}

type profileLoaderByRelFile struct{}

func (h *profileLoaderByRelFile) Execute(testName string) error {
results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud.yml")
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to load template with id")
}
if len(results) <= 10 {
return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
}
return nil
}

type profileLoaderById struct{}

func (h *profileLoaderById) Execute(testName string) error {
results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud")
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to load template with id")
}
if len(results) <= 10 {
return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
}
return nil
}

// this profile with load kevs
type customProfileLoader struct{}

func (h *customProfileLoader) Execute(filepath string) error {
results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", filepath)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to load template with id")
}
if len(results) < 1 {
return fmt.Errorf("incorrect result: expected more results than %d, got %v", 1, len(results))
}
return nil
}
64 changes: 59 additions & 5 deletions cmd/nuclei/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ import (
)

var (
cfgFile string
memProfile string // optional profile file path
options = &types.Options{}
cfgFile string
templateProfile string
memProfile string // optional profile file path
options = &types.Options{}
)

func main() {
Expand Down Expand Up @@ -199,7 +200,7 @@ on extensive configurability, massive extensibility and ease of use.`)
*/

flagSet.CreateGroup("input", "Target",
flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.StringSliceOptions),
flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions),
flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"),
Expand All @@ -225,6 +226,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"),
flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"),
flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),
Expand Down Expand Up @@ -270,6 +272,8 @@ on extensive configurability, massive extensibility and ease of use.`)

flagSet.CreateGroup("configs", "Configurations",
flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
flagSet.StringVarP(&templateProfile, "profile", "tp", "", "template profile config file to run"),
flagSet.BoolVarP(&options.ListTemplateProfiles, "profile-list", "tpl", false, "list community template profiles"),
flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"),
flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
Expand Down Expand Up @@ -409,7 +413,8 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CreateGroup("cloud", "Cloud",
flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"),
flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"),
flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),
)

flagSet.CreateGroup("Authentication", "Authentication",
Expand Down Expand Up @@ -497,6 +502,34 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
}

defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
if templateProfile != "" {
if filepath.Ext(templateProfile) == "" {
if tp := findProfilePathById(templateProfile, defaultProfilesPath); tp != "" {
templateProfile = tp
} else {
gologger.Fatal().Msgf("'%s' is not a profile-id or profile path", templateProfile)
}
}
if !filepath.IsAbs(templateProfile) {
if filepath.Dir(templateProfile) == "profiles" {
defaultProfilesPath = filepath.Join(config.DefaultConfig.GetTemplateDir())
}
currentDir, err := os.Getwd()
if err == nil && fileutil.FileExists(filepath.Join(currentDir, templateProfile)) {
templateProfile = filepath.Join(currentDir, templateProfile)
} else {
templateProfile = filepath.Join(defaultProfilesPath, templateProfile)
}
}
if !fileutil.FileExists(templateProfile) {
gologger.Fatal().Msgf("given template profile file '%s' does not exist", templateProfile)
}
if err := flagSet.MergeConfigFile(templateProfile); err != nil {
gologger.Fatal().Msgf("Could not read template profile: %s\n", err)
}
}

if len(options.SecretsFile) > 0 {
for _, secretFile := range options.SecretsFile {
if !fileutil.FileExists(secretFile) {
Expand Down Expand Up @@ -622,6 +655,27 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
os.Exit(0)
}

func findProfilePathById(profileId, templatesDir string) string {
var profilePath string
err := filepath.WalkDir(templatesDir, func(iterItem string, d fs.DirEntry, err error) error {
ext := filepath.Ext(iterItem)
isYaml := ext == extensions.YAML || ext == extensions.YML
if err != nil || d.IsDir() || !isYaml {
// skip non yaml files
return nil
}
if strings.TrimSuffix(filepath.Base(iterItem), ext) == profileId {
profilePath = iterItem
return fmt.Errorf("FOUND")
}
return nil
})
if err != nil && err.Error() != "FOUND" {
gologger.Error().Msgf("%s\n", err)
}
return profilePath
}

func init() {
// print stacktrace of errors in debug mode
if strings.EqualFold(os.Getenv("DEBUG"), "true") {
Expand Down
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.0.69
github.com/projectdiscovery/fastdialer v0.0.71
github.com/projectdiscovery/hmap v0.0.41
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.47
github.com/projectdiscovery/retryabledns v1.0.58
github.com/projectdiscovery/retryablehttp-go v1.0.58
github.com/projectdiscovery/retryabledns v1.0.59
github.com/projectdiscovery/retryablehttp-go v1.0.60
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
Expand Down Expand Up @@ -93,9 +93,9 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.48
github.com/projectdiscovery/useragent v0.0.49
github.com/projectdiscovery/utils v0.0.92
github.com/projectdiscovery/wappalyzergo v0.0.120
github.com/projectdiscovery/wappalyzergo v0.0.122
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
Expand Down
20 changes: 10 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU=
github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8=
github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4=
github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
Expand Down Expand Up @@ -875,10 +875,10 @@ github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5
github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4=
github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0=
github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ=
github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8=
github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
Expand All @@ -887,12 +887,12 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
github.com/projectdiscovery/useragent v0.0.48 h1:ITygElwcY9FlOt0F65kcW/oAALNr1nQOtO3kR9lRzaY=
github.com/projectdiscovery/useragent v0.0.48/go.mod h1:ahQMoWlVNFVQxjHOKqOPHJJQ7Ovz1zJZuJbBsAwIcOw=
github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960=
github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c=
github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
Expand Down
2 changes: 2 additions & 0 deletions integration_tests/profile-loader/basic.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
tags:
- kev
27 changes: 24 additions & 3 deletions internal/pdcp/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"io"
"net/http"
"net/url"
"regexp"
"sync/atomic"
"time"

Expand All @@ -27,9 +28,13 @@ const (
appendEndpoint = "/v1/scans/%s/import"
flushTimer = time.Duration(1) * time.Minute
MaxChunkSize = 1024 * 1024 * 4 // 4 MB
xidRe = `^[a-z0-9]{20}$`
)

var _ output.Writer = &UploadWriter{}
var (
xidRegex = regexp.MustCompile(xidRe)
_ output.Writer = &UploadWriter{}
)

// UploadWriter is a writer that uploads its output to pdcp
// server to enable web dashboard and more
Expand All @@ -41,6 +46,7 @@ type UploadWriter struct {
cancel context.CancelFunc
done chan struct{}
scanID string
scanName string
counter atomic.Int32
}

Expand Down Expand Up @@ -86,8 +92,17 @@ func NewUploadWriter(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*Upl
}

// SetScanID sets the scan id for the upload writer
func (u *UploadWriter) SetScanID(id string) {
func (u *UploadWriter) SetScanID(id string) error {
if !xidRegex.MatchString(id) {
return fmt.Errorf("invalid scan id provided")
}
u.scanID = id
return nil
}

// SetScanName sets the scan name for the upload writer
func (u *UploadWriter) SetScanName(name string) {
u.scanName = name
}

func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
Expand Down Expand Up @@ -220,7 +235,13 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request")
}
// add pdtm meta params
req.URL.RawQuery = updateutils.GetpdtmParams(config.Version)
req.URL.Params.Merge(updateutils.GetpdtmParams(config.Version))
// if it is upload endpoint also include name if it exists
if u.scanName != "" && req.URL.Path == uploadEndpoint {
req.URL.Params.Add("name", u.scanName)
}
req.URL.Update()

req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", "application/json")
Expand Down
27 changes: 27 additions & 0 deletions internal/runner/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package runner
import (
"bufio"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
Expand All @@ -25,6 +26,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/extensions"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
fileutil "github.com/projectdiscovery/utils/file"
Expand Down Expand Up @@ -74,6 +76,31 @@ func ParseOptions(options *types.Options) {
}
os.Exit(0)
}

defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
if options.ListTemplateProfiles {
gologger.Print().Msgf(
"\nListing available %v nuclei template profiles for %v",
config.DefaultConfig.TemplateVersion,
config.DefaultConfig.TemplatesDirectory,
)
templatesRootDir := config.DefaultConfig.GetTemplateDir()
err := filepath.WalkDir(defaultProfilesPath, func(iterItem string, d fs.DirEntry, err error) error {
ext := filepath.Ext(iterItem)
isYaml := ext == extensions.YAML || ext == extensions.YML
if err != nil || d.IsDir() || !isYaml {
return nil
}
if profileRelPath, err := filepath.Rel(templatesRootDir, iterItem); err == nil {
gologger.Print().Msgf("%s (%s)\n", profileRelPath, strings.TrimSuffix(filepath.Base(iterItem), ext))
}
return nil
})
if err != nil {
gologger.Error().Msgf("%s\n", err)
}
os.Exit(0)
}
if options.StoreResponseDir != DefaultDumpTrafficOutputFolder && !options.StoreResponse {
gologger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
options.StoreResponse = true
Expand Down
Loading

0 comments on commit 8e81d7d

Please sign in to comment.