feat: misc review changes

Ice3man543 · Ice3man543 · commit 9f4b89ac8434 · 2024-12-16T21:10:01.000+05:30
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
@@ -368,7 +368,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
 		flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"),
 		flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"),
-		flagSet.BoolVarP(&options.DASTReport, "dast-report", "drg", false, "write dast scan report to file"),
+		flagSet.BoolVarP(&options.DASTReport, "dast-report", "dtr", false, "write dast scan report to file"),
 		flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"),
 		flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"),
 		flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"),
diff --git a/go.sum b/go.sum
@@ -864,8 +864,6 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7
 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
 github.com/projectdiscovery/dsl v0.3.3 h1:4Ij5S86cHlb6xFrS7+5zAiJPeBt5h970XBTHqeTkpyU=
 github.com/projectdiscovery/dsl v0.3.3/go.mod h1:DAjSeaogLM9f0Ves2zDc/vbJrfcv+kEmS51p0dLLaPI=
-github.com/projectdiscovery/fastdialer v0.2.11 h1:DTx2vJ3tytv34wDe+Oh72L7v9pZWhzNGFJgwheN0n1Q=
-github.com/projectdiscovery/fastdialer v0.2.11/go.mod h1:jjDMLl+hnKoSSP82eWPxn8U+KivlWqf/o3pSz4n4dik=
 github.com/projectdiscovery/fastdialer v0.2.13 h1:5XzSv0hwITzRAMwyoJ9GFZSTVtaI4jmwER968TbDLbI=
 github.com/projectdiscovery/fastdialer v0.2.13/go.mod h1:T1EaYHbWmCnVHSYz12nAjnHMNFEfGMLLw37cb0k1X3A=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
diff --git a/internal/runner/options.go b/internal/runner/options.go
@@ -171,6 +171,11 @@ func ValidateOptions(options *types.Options) error {
 	if options.Validate {
 		validateTemplatePaths(config.DefaultConfig.TemplatesDirectory, options.Templates, options.Workflows)
 	}
+	if options.DAST {
+		if err := validateDASTOptions(options); err != nil {
+			return err
+		}
+	}
 
 	// Verify if any of the client certificate options were set since it requires all three to work properly
 	if options.HasClientCertificates() {
@@ -274,6 +279,14 @@ func validateMissingGitLabOptions(options *types.Options) []string {
 	return missing
 }
 
+func validateDASTOptions(options *types.Options) error {
+	// Ensure the DAST server token meets minimum length requirement
+	if len(options.DASTServerToken) > 0 && len(options.DASTServerToken) < 16 {
+		return fmt.Errorf("DAST server token must be at least 16 characters long")
+	}
+	return nil
+}
+
 func createReportingOptions(options *types.Options) (*reporting.Options, error) {
 	var reportingOptions = &reporting.Options{}
 	if options.ReportingConfig != "" {
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
@@ -305,7 +305,7 @@ func New(options *types.Options) (*Runner, error) {
 		return nil, errors.Wrap(err, "could not create output file")
 	}
 	if runner.fuzzStats != nil {
-		outputWriter.RequestHook = func(request *output.JSONLogRequest) {
+		outputWriter.JSONLogRequestHook = func(request *output.JSONLogRequest) {
 			if request.Error == "none" || request.Error == "" {
 				return
 			}
@@ -687,9 +687,11 @@ func (r *Runner) RunEnumeration() error {
 	}, "")
 
 	if r.dastServer != nil {
-		if err := r.dastServer.Start(); err != nil {
-			r.dastServer.Start()
-		}
+		go func() {
+			if err := r.dastServer.Start(); err != nil {
+				gologger.Error().Msgf("could not start dast server: %v", err)
+			}
+		}()
 	}
 
 	enumeration := false
diff --git a/internal/server/nuclei_sdk.go b/internal/server/nuclei_sdk.go
@@ -121,7 +121,7 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) {
 	}
 	store, err := loader.New(loaderConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "Could not create loadeopts.")
+		return nil, errors.Wrap(err, "Could not create loader options.")
 	}
 	store.Load()
 
@@ -143,7 +143,7 @@ type proxifyRequest struct {
 	} `json:"request"`
 }
 
-func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error {
+func (n *nucleiExecutor) ExecuteScan(target PostRequestsHandlerRequest) error {
 	finalTemplates := []*templates.Template{}
 	finalTemplates = append(finalTemplates, n.store.Templates()...)
 	finalTemplates = append(finalTemplates, n.store.Workflows()...)
@@ -178,6 +178,9 @@ func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error {
 	if err != nil {
 		return errors.Wrap(err, "could not create input provider")
 	}
+
+	// We don't care about the result as its a boolean
+	// stating whether we got matches or not
 	_ = n.engine.ExecuteScanWithOpts(context.Background(), finalTemplates, inputProvider, true)
 	return nil
 }
diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go
@@ -8,7 +8,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
 )
 
-func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) {
+func (s *DASTServer) consumeTaskRequest(req PostRequestsHandlerRequest) {
 	defer s.endpointsInQueue.Add(-1)
 
 	parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL)
@@ -29,7 +29,7 @@ func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) {
 		return
 	}
 
-	inScope, err := s.scopeManager.Validate(parsedReq.URL.URL, "")
+	inScope, err := s.scopeManager.Validate(parsedReq.URL.URL)
 	if err != nil {
 		gologger.Warning().Msgf("Could not validate scope: %s\n", err)
 		return
diff --git a/internal/server/scope/scope.go b/internal/server/scope/scope.go
@@ -39,7 +39,7 @@ func NewManager(inScope, outOfScope []string) (*Manager, error) {
 }
 
 // Validate returns true if the URL matches scope rules
-func (m *Manager) Validate(URL *url.URL, rootHostname string) (bool, error) {
+func (m *Manager) Validate(URL *url.URL) (bool, error) {
 	if m.noScope {
 		return true, nil
 	}
diff --git a/internal/server/scope/scope_test.go b/internal/server/scope/scope_test.go
@@ -13,12 +13,12 @@ func TestManagerValidate(t *testing.T) {
 		require.NoError(t, err, "could not create scope manager")
 
 		parsed, _ := urlutil.Parse("https://test.com/index.php/example")
-		validated, err := manager.Validate(parsed.URL, "test.com")
+		validated, err := manager.Validate(parsed.URL)
 		require.NoError(t, err, "could not validate url")
 		require.True(t, validated, "could not get correct in-scope validation")
 
 		parsed, _ = urlutil.Parse("https://test.com/logout.php")
-		validated, err = manager.Validate(parsed.URL, "another.com")
+		validated, err = manager.Validate(parsed.URL)
 		require.NoError(t, err, "could not validate url")
 		require.False(t, validated, "could not get correct out-scope validation")
 	})
diff --git a/internal/server/server.go b/internal/server/server.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html/template"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -103,8 +104,8 @@ func New(options *Options) (*DASTServer, error) {
 		builder.WriteString(" (with token)")
 	}
 	gologger.Info().Msgf("%s", builder.String())
-	gologger.Info().Msgf("Connection URL: %s", server.buildConnectionURL())
-	gologger.Info().Msgf("Stats UI URL: %s", server.buildStatsURL())
+	gologger.Info().Msgf("Connection URL: %s", server.buildURL("/requests"))
+	gologger.Info().Msgf("Stats UI URL: %s", server.buildURL("/stats"))
 
 	return server, nil
 }
@@ -118,7 +119,7 @@ func NewStatsServer(fuzzStatsDB *stats.Tracker) (*DASTServer, error) {
 		},
 	}
 	server.setupHandlers(true)
-	gologger.Info().Msgf("Stats UI URL: %s", server.buildStatsURL())
+	gologger.Info().Msgf("Stats UI URL: %s", server.buildURL("/stats"))
 
 	return server, nil
 }
@@ -129,20 +130,20 @@ func (s *DASTServer) Close() {
 	s.tasksPool.StopAndWaitFor(1 * time.Minute)
 }
 
-func (s *DASTServer) buildConnectionURL() string {
-	url := fmt.Sprintf("http://%s/requests", s.options.Address)
+func (s *DASTServer) buildURL(endpoint string) string {
+	values := make(url.Values)
 	if s.options.Token != "" {
-		url += "?token=" + s.options.Token
+		values.Set("token", s.options.Token)
 	}
-	return url
-}
 
-func (s *DASTServer) buildStatsURL() string {
-	url := fmt.Sprintf("http://%s/stats", s.options.Address)
-	if s.options.Token != "" {
-		url += "?token=" + s.options.Token
+	// Use url.URL struct to safely construct the URL
+	u := &url.URL{
+		Scheme:   "http",
+		Host:     s.options.Address,
+		Path:     endpoint,
+		RawQuery: values.Encode(),
 	}
-	return url
+	return u.String()
 }
 
 func (s *DASTServer) setupHandlers(onlyStats bool) {
@@ -186,13 +187,13 @@ func (s *DASTServer) Start() error {
 }
 
 // PostReuestsHandlerRequest is the request body for the /requests POST handler.
-type PostReuestsHandlerRequest struct {
+type PostRequestsHandlerRequest struct {
 	RawHTTP string `json:"raw_http"`
 	URL     string `json:"url"`
 }
 
 func (s *DASTServer) handleRequest(c echo.Context) error {
-	var req PostReuestsHandlerRequest
+	var req PostRequestsHandlerRequest
 	if err := c.Bind(&req); err != nil {
 		fmt.Printf("Error binding request: %s\n", err)
 		return err
@@ -246,7 +247,7 @@ func (s *DASTServer) getStats() (StatsResponse, error) {
 		DASTServerInfo: DASTServerInfo{
 			NucleiVersion:         config.Version,
 			NucleiTemplateVersion: cfg.TemplateVersion,
-			NucleiDastServerAPI:   s.buildConnectionURL(),
+			NucleiDastServerAPI:   s.buildURL("/requests"),
 			ServerAuthEnabled:     s.options.Token != "",
 		},
 		DASTScanStartTime: s.startTime,
diff --git a/internal/server/templates/index.html b/internal/server/templates/index.html
@@ -3,7 +3,7 @@
 <head>
     <meta charset="UTF-8">
     <title>DAST Scan Report</title>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
+    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/bootstrap-icons/1.11.3/font/bootstrap-icons.css" integrity="sha512-ywmPbuxGS4cJ7GxwCX+bCJweeext047ZYU2HP52WWKbpJnF4/Zzfr2Bo19J4CWPXZmleVusQ9d//RB5bq0RP7w==" crossorigin="anonymous" referrerpolicy="no-referrer" />
     <style>
         @import url('https://fonts.googleapis.com/css2?family=Geist+Mono:wght@400;500&display=swap');
 
diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go
@@ -62,18 +62,24 @@ func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.Adaptive
 }
 
 func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
+	changedValues := make(map[string]int)
 	if w.config.TypeConcurrency != config.TypeConcurrency {
 		w.config.TypeConcurrency = config.TypeConcurrency
+		changedValues["TypeConcurrency"] = config.TypeConcurrency
 	}
 	if w.config.HeadlessTypeConcurrency != config.HeadlessTypeConcurrency {
 		w.config.HeadlessTypeConcurrency = config.HeadlessTypeConcurrency
 	}
 	if w.config.InputConcurrency != config.InputConcurrency {
 		w.config.InputConcurrency = config.InputConcurrency
+		changedValues["InputConcurrency"] = config.InputConcurrency
 	}
 	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
 		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
 	}
+	if len(changedValues) > 0 {
+		gologger.Info().Msgf("Workpool configuration changed: %v", changedValues)
+	}
 	w.Refresh(context.Background())
 }
 
diff --git a/pkg/fuzz/analyzers/time/time_delay.go b/pkg/fuzz/analyzers/time/time_delay.go
@@ -33,7 +33,9 @@ import (
 
 type timeDelayRequestSender func(delay int) (float64, error)
 
-type requstsSentMetadata struct {
+// requestsSentMetadata is used to store the delay requested
+// and delay received for each request
+type requestsSentMetadata struct {
 	delay         int
 	delayReceived float64
 }
@@ -57,7 +59,7 @@ func checkTimingDependency(
 	regression := newSimpleLinearRegression()
 	requestsLeft := requestsLimit
 
-	var requestsSent []requstsSentMetadata
+	var requestsSent []requestsSentMetadata
 	for {
 		if requestsLeft <= 0 {
 			break
@@ -74,7 +76,7 @@ func checkTimingDependency(
 		if delayRecieved < baselineDelay+float64(highSleepTimeSeconds)*0.8 {
 			return false, "", nil
 		}
-		requestsSent = append(requestsSent, requstsSentMetadata{
+		requestsSent = append(requestsSent, requestsSentMetadata{
 			delay:         highSleepTimeSeconds,
 			delayReceived: delayRecieved,
 		})
@@ -91,7 +93,7 @@ func checkTimingDependency(
 		}
 		requestsLeft = requestsLeft - 2
 
-		requestsSent = append(requestsSent, requstsSentMetadata{
+		requestsSent = append(requestsSent, requestsSentMetadata{
 			delay:         int(DefaultLowSleepTimeSeconds),
 			delayReceived: delayRecievedSecond,
 		})
@@ -101,8 +103,9 @@ func checkTimingDependency(
 	if result {
 		var resultReason strings.Builder
 		resultReason.WriteString(fmt.Sprintf(
-			"[time_delay] made %d requests successfully, with a regression slope of %.2f and correlation %.2f",
+			"[time_delay] made %d requests (baseline: %.2fs) successfully, with a regression slope of %.2f and correlation %.2f",
 			requestsLimit,
+			baselineDelay,
 			regression.slope,
 			regression.correlation,
 		))
diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go
@@ -41,6 +41,10 @@ type HttpInputProvider struct {
 }
 
 // NewHttpInputProvider creates a new input provider for nuclei from a file
+// or an input string
+//
+// The first preference is given to input file if provided
+// otherwise it will use the input string
 func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, error) {
 	var format formats.Format
 	for _, provider := range providersList {
@@ -77,6 +81,9 @@ func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, err
 	if err != nil {
 		return nil, errors.Wrap(err, "could not read input file")
 	}
+	if len(data) == 0 {
+		return nil, errors.New("input file is empty")
+	}
 
 	parseErr := format.Parse(bytes.NewReader(data), func(request *types.RequestResponse) bool {
 		count++
diff --git a/pkg/output/output.go b/pkg/output/output.go
@@ -74,7 +74,9 @@ type StandardWriter struct {
 	AddNewLinesOutputFile bool // by default this is only done for stdout
 	KeysToRedact          []string
 
-	RequestHook func(*JSONLogRequest)
+	// JSONLogRequestHook is a hook that can be used to log request/response
+	// when using custom server code with output
+	JSONLogRequestHook func(*JSONLogRequest)
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -350,7 +352,7 @@ type JSONLogRequest struct {
 
 // Request writes a log the requests trace log
 func (w *StandardWriter) Request(templatePath, input, requestType string, requestErr error) {
-	if w.traceFile == nil && w.errorFile == nil && w.RequestHook == nil {
+	if w.traceFile == nil && w.errorFile == nil && w.JSONLogRequestHook == nil {
 		return
 	}
 	request := &JSONLogRequest{
@@ -400,8 +402,8 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 		request.Address = val.String()
 	}
 
-	if w.RequestHook != nil {
-		w.RequestHook(request)
+	if w.JSONLogRequestHook != nil {
+		w.JSONLogRequestHook(request)
 	}
 
 	data, err := jsoniter.Marshal(request)

Original file line number	Diff line number	Diff line change
`@@ -121,7 +121,7 @@ func newNucleiExecutor(opts NucleiExecutorOptions) (nucleiExecutor, error) {`
`121`	`121`	`}`
`122`	`122`	`store, err := loader.New(loaderConfig)`
`123`	`123`	`if err != nil {`
`124`		`- return nil, errors.Wrap(err, "Could not create loadeopts.")`
	`124`	`+ return nil, errors.Wrap(err, "Could not create loader options.")`
`125`	`125`	`}`
`126`	`126`	`store.Load()`
`127`	`127`
`@@ -143,7 +143,7 @@ type proxifyRequest struct {`
`143`	`143`	} `json:"request"`
`144`	`144`	`}`
`145`	`145`
`146`		`-func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error {`
	`146`	`+func (n *nucleiExecutor) ExecuteScan(target PostRequestsHandlerRequest) error {`
`147`	`147`	`finalTemplates := []*templates.Template{}`
`148`	`148`	`finalTemplates = append(finalTemplates, n.store.Templates()...)`
`149`	`149`	`finalTemplates = append(finalTemplates, n.store.Workflows()...)`
`@@ -178,6 +178,9 @@ func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error {`
`178`	`178`	`if err != nil {`
`179`	`179`	`return errors.Wrap(err, "could not create input provider")`
`180`	`180`	`}`
	`181`	`+`
	`182`	`+ // We don't care about the result as its a boolean`
	`183`	`+ // stating whether we got matches or not`
`181`	`184`	`_ = n.engine.ExecuteScanWithOpts(context.Background(), finalTemplates, inputProvider, true)`
`182`	`185`	`return nil`
`183`	`186`	`}`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ import (`
`8`	`8`	`"github.com/projectdiscovery/nuclei/v3/pkg/input/types"`
`9`	`9`	`)`
`10`	`10`
`11`		`-func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) {`
	`11`	`+func (s *DASTServer) consumeTaskRequest(req PostRequestsHandlerRequest) {`
`12`	`12`	`defer s.endpointsInQueue.Add(-1)`
`13`	`13`
`14`	`14`	`parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL)`
`@@ -29,7 +29,7 @@ func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) {`
`29`	`29`	`return`
`30`	`30`	`}`
`31`	`31`
`32`		`- inScope, err := s.scopeManager.Validate(parsedReq.URL.URL, "")`
	`32`	`+ inScope, err := s.scopeManager.Validate(parsedReq.URL.URL)`
`33`	`33`	`if err != nil {`
`34`	`34`	`gologger.Warning().Msgf("Could not validate scope: %s\n", err)`
`35`	`35`	`return`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ func NewManager(inScope, outOfScope []string) (*Manager, error) {`
`39`	`39`	`}`
`40`	`40`
`41`	`41`	`// Validate returns true if the URL matches scope rules`
`42`		`-func (m Manager) Validate(URL url.URL, rootHostname string) (bool, error) {`
	`42`	`+func (m Manager) Validate(URL url.URL) (bool, error) {`
`43`	`43`	`if m.noScope {`
`44`	`44`	`return true, nil`
`45`	`45`	`}`