Merge remote-tracking branch 'origin'

Author: ehsandeep

cmd/integration-test/profile-loader.go

@@ -16,38 +16,39 @@ var profileLoaderTestcases = []TestCaseInfo{
 type profileLoaderByRelFile struct{}
 
 func (h *profileLoaderByRelFile) Execute(testName string) error {
-	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "kev.yml")
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud.yml")
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
 	}
-	if len(results) < 267 {
-		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 267, len(results))
+	if len(results) <= 10 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
 	}
 	return nil
 }
 
 type profileLoaderById struct{}
 
 func (h *profileLoaderById) Execute(testName string) error {
-	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "kev")
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud")
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
 	}
-	if len(results) < 267 {
-		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 267, len(results))
+	if len(results) <= 10 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
 	}
 	return nil
 }
 
+// this profile with load kevs
 type customProfileLoader struct{}
 
 func (h *customProfileLoader) Execute(filepath string) error {
 	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", filepath)
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
 	}
-	if len(results) < 267 {
-		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 267, len(results))
+	if len(results) < 1 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 1, len(results))
 	}
 	return nil
 }

cmd/nuclei/main.go

@@ -200,7 +200,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	*/
 
 	flagSet.CreateGroup("input", "Target",
-		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.StringSliceOptions),
+		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
 		flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"),
@@ -414,6 +414,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
 		flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
 		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"),
+		flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),
 	)
 
 	flagSet.CreateGroup("Authentication", "Authentication",

go.mod

@@ -93,7 +93,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/useragent v0.0.48
+	github.com/projectdiscovery/useragent v0.0.49
 	github.com/projectdiscovery/utils v0.0.92
 	github.com/projectdiscovery/wappalyzergo v0.0.120
 	github.com/redis/go-redis/v9 v9.1.0

go.sum

@@ -887,8 +887,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/useragent v0.0.48 h1:ITygElwcY9FlOt0F65kcW/oAALNr1nQOtO3kR9lRzaY=
-github.com/projectdiscovery/useragent v0.0.48/go.mod h1:ahQMoWlVNFVQxjHOKqOPHJJQ7Ovz1zJZuJbBsAwIcOw=
+github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
+github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
 github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
 github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
 github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960=

internal/pdcp/writer.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"regexp"
 	"sync/atomic"
 	"time"
 
@@ -27,9 +28,13 @@ const (
 	appendEndpoint = "/v1/scans/%s/import"
 	flushTimer     = time.Duration(1) * time.Minute
 	MaxChunkSize   = 1024 * 1024 * 4 // 4 MB
+	xidRe          = `^[a-z0-9]{20}$`
 )
 
-var _ output.Writer = &UploadWriter{}
+var (
+	xidRegex               = regexp.MustCompile(xidRe)
+	_        output.Writer = &UploadWriter{}
+)
 
 // UploadWriter is a writer that uploads its output to pdcp
 // server to enable web dashboard and more
@@ -41,6 +46,7 @@ type UploadWriter struct {
 	cancel    context.CancelFunc
 	done      chan struct{}
 	scanID    string
+	scanName  string
 	counter   atomic.Int32
 }
 
@@ -86,8 +92,17 @@ func NewUploadWriter(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*Upl
 }
 
 // SetScanID sets the scan id for the upload writer
-func (u *UploadWriter) SetScanID(id string) {
+func (u *UploadWriter) SetScanID(id string) error {
+	if !xidRegex.MatchString(id) {
+		return fmt.Errorf("invalid scan id provided")
+	}
 	u.scanID = id
+	return nil
+}
+
+// SetScanName sets the scan name for the upload writer
+func (u *UploadWriter) SetScanName(name string) {
+	u.scanName = name
 }
 
 func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
@@ -220,7 +235,13 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
 		return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request")
 	}
 	// add pdtm meta params
-	req.URL.RawQuery = updateutils.GetpdtmParams(config.Version)
+	req.URL.Params.Merge(updateutils.GetpdtmParams(config.Version))
+	// if it is upload endpoint also include name if it exists
+	if u.scanName != "" && req.URL.Path == uploadEndpoint {
+		req.URL.Params.Add("name", u.scanName)
+	}
+	req.URL.Update()
+
 	req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
 	req.Header.Set("Content-Type", "application/octet-stream")
 	req.Header.Set("Accept", "application/json")

internal/runner/runner.go

@@ -414,7 +414,11 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer {
 		return writer
 	}
 	if r.options.ScanID != "" {
-		uploadWriter.SetScanID(r.options.ScanID)
+		// ignore and use empty scan id if invalid
+		_ = uploadWriter.SetScanID(r.options.ScanID)
+	}
+	if r.options.ScanName != "" {
+		uploadWriter.SetScanName(r.options.ScanName)
 	}
 	return output.NewMultiWriter(writer, uploadWriter)
 }

lib/multi.go

@@ -111,11 +111,11 @@ func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *outpu
 	e.eng.resultCallbacks = []func(*output.ResultEvent){callback}
 }
 
-// ExecuteNucleiWithOpts executes templates on targets and calls callback on each result(only if results are found)
+// ExecuteNucleiWithOptsCtx executes templates on targets and calls callback on each result(only if results are found)
 // This method can be called concurrently and it will use some global resources but can be runned parallelly
 // by invoking this method with different options and targets
 // Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options
-func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, targets []string, opts ...NucleiSDKOptions) error {
 	baseOpts := *e.eng.opts
 	tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe}
 	for _, option := range opts {
@@ -163,6 +163,12 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 	return nil
 }
 
+// ExecuteNucleiWithOpts is same as ExecuteNucleiWithOptsCtx but with default context
+// This is a placeholder and will be deprecated in future major release
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+	return e.ExecuteNucleiWithOptsCtx(context.Background(), targets, opts...)
+}
+
 // Close all resources used by nuclei engine
 func (e *ThreadSafeNucleiEngine) Close() {
 	e.eng.Close()

lib/sdk.go

@@ -219,8 +219,9 @@ func (e *NucleiEngine) Close() {
 	}
 }
 
-// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
-func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+// ExecuteCallbackWithCtx executes templates on targets and calls callback on each result(only if results are found)
+// enable matcher-status option if you expect this callback to be called for all results regardless if it matched or not
+func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...func(event *output.ResultEvent)) error {
 	if !e.templatesLoaded {
 		_ = e.LoadAllTemplates()
 	}
@@ -244,10 +245,18 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	return nil
 }
 
+// ExecuteWithCallback is same as ExecuteCallbackWithCtx but with default context
+// Note this is deprecated and will be removed in future major release
+func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+	return e.ExecuteCallbackWithCtx(context.Background(), callback...)
+}
+
+// Options return nuclei Type Options
 func (e *NucleiEngine) Options() *types.Options {
 	return e.opts
 }
 
+// Engine returns core Executer of nuclei
 func (e *NucleiEngine) Engine() *core.Engine {
 	return e.engine
 }

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.6`
+	Version = `v3.2.7`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/core/executors.go

@@ -162,7 +162,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
 	// wp is workpool that contains different waitgroups for
 	// headless and non-headless templates
 	// global waitgroup should not be used here
-	wp := e.WorkPool()
+	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
 		select {