feat: issue tracker URLs in JSON + misc fixes (#4855)

* feat: issue tracker URLs in JSON + misc fixes

* misc changes

* feat: status update support for issues

* feat: report metadata generation hook support

* feat: added CLI summary of tickets created

* misc changes

Author: Ice3man543

cmd/integration-test/library.go

@@ -69,7 +69,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	defer cache.Close()
 
 	mockProgress := &testutils.MockProgressClient{}
-	reportingClient, err := reporting.New(&reporting.Options{}, "")
+	reportingClient, err := reporting.New(&reporting.Options{}, "", false)
 	if err != nil {
 		return nil, err
 	}

internal/runner/options.go

@@ -290,12 +290,14 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose || options.Validate {
-		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
-	}
 	if options.Debug || options.DebugRequests || options.DebugResponse {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
 	}
+	// Debug takes precedence before verbose
+	// because debug is a lower logging level.
+	if options.Verbose || options.Validate {
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+	}
 	if options.NoColor {
 		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
 	}

internal/runner/runner.go

@@ -190,7 +190,7 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if reportingOptions != nil {
-		client, err := reporting.New(reportingOptions, options.ReportingDB)
+		client, err := reporting.New(reportingOptions, options.ReportingDB, false)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not create issue reporting client")
 		}

lib/sdk_private.go

@@ -128,7 +128,7 @@ func (e *NucleiEngine) init() error {
 		return err
 	}
 	// we don't support reporting config in sdk mode
-	if e.rc, err = reporting.New(&reporting.Options{}, ""); err != nil {
+	if e.rc, err = reporting.New(&reporting.Options{}, "", false); err != nil {
 		return err
 	}
 	e.interactshOpts.IssuesClient = e.rc

pkg/output/output.go

@@ -165,10 +165,20 @@ type ResultEvent struct {
 	// Lines is the line count for the specified match
 	Lines []int `json:"matched-line,omitempty"`
 
+	// IssueTrackers is the metadata for issue trackers
+	IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"`
+
 	FileToIndexPosition map[string]int `json:"-"`
 	Error               string         `json:"error,omitempty"`
 }
 
+type IssueTrackerMetadata struct {
+	// IssueID is the ID of the issue created
+	IssueID string `json:"id,omitempty"`
+	// IssueURL is the URL of the issue created
+	IssueURL string `json:"url,omitempty"`
+}
+
 // NewStandardWriter creates a new output writer based on user configurations
 func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
 	resumeBool := false

pkg/protocols/common/helpers/writer/writer.go

@@ -17,19 +17,18 @@ func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progre
 	}
 	var matched bool
 	for _, result := range data.Results {
+		if issuesClient != nil {
+			if err := issuesClient.CreateIssue(result); err != nil {
+				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
+			}
+		}
 		if err := output.Write(result); err != nil {
 			gologger.Warning().Msgf("Could not write output event: %s\n", err)
 		}
 		if !matched {
 			matched = true
 		}
 		progress.IncrementMatched()
-
-		if issuesClient != nil {
-			if err := issuesClient.CreateIssue(result); err != nil {
-				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
-			}
-		}
 	}
 	return matched
 }

pkg/reporting/client.go

@@ -11,5 +11,6 @@ type Client interface {
 	Close()
 	Clear()
 	CreateIssue(event *output.ResultEvent) error
+	CloseIssue(event *output.ResultEvent) error
 	GetReportingOptions() *Options
 }

pkg/reporting/format/format_utils.go

@@ -34,6 +34,13 @@ func GetMatchedTemplateName(event *output.ResultEvent) string {
 	return matchedTemplateName
 }
 
+type reportMetadataEditorHook func(event *output.ResultEvent, formatter ResultFormatter) string
+
+var (
+	// ReportGenerationMetadataHooks are the hooks for adding metadata to the report
+	ReportGenerationMetadataHooks []reportMetadataEditorHook
+)
+
 func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter, omitRaw bool) string {
 	template := GetMatchedTemplateName(event)
 	builder := &bytes.Buffer{}
@@ -137,6 +144,12 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte
 
 	builder.WriteString("\n" + formatter.CreateHorizontalLine() + "\n")
 	builder.WriteString(fmt.Sprintf("Generated by %s", formatter.CreateLink("Nuclei "+config.Version, "https://github.com/projectdiscovery/nuclei")))
+
+	if len(ReportGenerationMetadataHooks) > 0 {
+		for _, hook := range ReportGenerationMetadataHooks {
+			builder.WriteString(hook(event, formatter))
+		}
+	}
 	data := builder.String()
 	return data
 }

pkg/reporting/reporting.go

@@ -1,8 +1,12 @@
 package reporting
 
 import (
+	"fmt"
 	"os"
+	"strings"
+	"sync/atomic"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	json_exporter "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
@@ -35,8 +39,12 @@ var (
 
 // Tracker is an interface implemented by an issue tracker
 type Tracker interface {
+	// Name returns the name of the tracker
+	Name() string
 	// CreateIssue creates an issue in the tracker
-	CreateIssue(event *output.ResultEvent) error
+	CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error)
+	// CloseIssue closes an issue in the tracker
+	CloseIssue(event *output.ResultEvent) error
 	// ShouldFilter determines if the event should be filtered out
 	ShouldFilter(event *output.ResultEvent) bool
 }
@@ -55,10 +63,17 @@ type ReportingClient struct {
 	exporters []Exporter
 	options   *Options
 	dedupe    *dedupe.Storage
+
+	stats map[string]*IssueTrackerStats
+}
+
+type IssueTrackerStats struct {
+	Created atomic.Int32
+	Failed  atomic.Int32
 }
 
 // New creates a new nuclei issue tracker reporting client
-func New(options *Options, db string) (Client, error) {
+func New(options *Options, db string, doNotDedupe bool) (Client, error) {
 	client := &ReportingClient{options: options}
 
 	if options.GitHub != nil {
@@ -142,6 +157,20 @@ func New(options *Options, db string) (Client, error) {
 		client.exporters = append(client.exporters, exporter)
 	}
 
+	if doNotDedupe {
+		return client, nil
+	}
+
+	client.stats = make(map[string]*IssueTrackerStats)
+	for _, tracker := range client.trackers {
+		trackerName := tracker.Name()
+
+		client.stats[trackerName] = &IssueTrackerStats{
+			Created: atomic.Int32{},
+			Failed:  atomic.Int32{},
+		}
+	}
+
 	storage, err := dedupe.New(db)
 	if err != nil {
 		return nil, err
@@ -195,7 +224,30 @@ func (c *ReportingClient) RegisterExporter(exporter Exporter) {
 
 // Close closes the issue tracker reporting client
 func (c *ReportingClient) Close() {
-	c.dedupe.Close()
+	// If we have stats for the trackers, print them
+	if len(c.stats) > 0 {
+		for _, tracker := range c.trackers {
+			trackerName := tracker.Name()
+
+			if stats, ok := c.stats[trackerName]; ok {
+				created := stats.Created.Load()
+				if created == 0 {
+					continue
+				}
+				var msgBuilder strings.Builder
+				msgBuilder.WriteString(fmt.Sprintf("%d %s tickets created successfully", created, trackerName))
+				failed := stats.Failed.Load()
+				if failed > 0 {
+					msgBuilder.WriteString(fmt.Sprintf(", %d failed", failed))
+				}
+				gologger.Info().Msgf(msgBuilder.String())
+			}
+		}
+	}
+
+	if c.dedupe != nil {
+		c.dedupe.Close()
+	}
 	for _, exporter := range c.exporters {
 		exporter.Close()
 	}
@@ -211,15 +263,37 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
 		return nil
 	}
 
-	unique, err := c.dedupe.Index(event)
+	var err error
+	unique := true
+	if c.dedupe != nil {
+		unique, err = c.dedupe.Index(event)
+	}
 	if unique {
+		event.IssueTrackers = make(map[string]output.IssueTrackerMetadata)
+
 		for _, tracker := range c.trackers {
 			// process tracker specific allow/deny list
 			if tracker.ShouldFilter(event) {
 				continue
 			}
-			if trackerErr := tracker.CreateIssue(event); trackerErr != nil {
+			trackerName := tracker.Name()
+			stats, statsOk := c.stats[trackerName]
+
+			reportData, trackerErr := tracker.CreateIssue(event)
+			if trackerErr != nil {
+				if statsOk {
+					_ = stats.Failed.Add(1)
+				}
 				err = multierr.Append(err, trackerErr)
+				continue
+			}
+			if statsOk {
+				_ = stats.Created.Add(1)
+			}
+
+			event.IssueTrackers[tracker.Name()] = output.IssueTrackerMetadata{
+				IssueID:  reportData.IssueID,
+				IssueURL: reportData.IssueURL,
 			}
 		}
 		for _, exporter := range c.exporters {
@@ -231,10 +305,25 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
 	return err
 }
 
+// CloseIssue closes an issue in the tracker
+func (c *ReportingClient) CloseIssue(event *output.ResultEvent) error {
+	for _, tracker := range c.trackers {
+		if tracker.ShouldFilter(event) {
+			continue
+		}
+		if err := tracker.CloseIssue(event); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (c *ReportingClient) GetReportingOptions() *Options {
 	return c.options
 }
 
 func (c *ReportingClient) Clear() {
-	c.dedupe.Clear()
+	if c.dedupe != nil {
+		c.dedupe.Clear()
+	}
 }

pkg/reporting/trackers/filters/filters.go

@@ -8,6 +8,13 @@ import (
 	sliceutil "github.com/projectdiscovery/utils/slice"
 )
 
+// CreateIssueResponse is a response to creating an issue
+// in a tracker
+type CreateIssueResponse struct {
+	IssueID  string `json:"issue_id"`
+	IssueURL string `json:"issue_url"`
+}
+
 // Filter filters the received event and decides whether to perform
 // reporting for it or not.
 type Filter struct {