omit raw from integrations (#4612)

dogancanbakir · web-flow · commit e102caec78d9 · 2024-01-27T04:06:25.000+05:30
* omit raw from integrations

* fix lint
diff --git a/internal/runner/options.go b/internal/runner/options.go
@@ -262,27 +262,28 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
 	}
 	if options.MarkdownExportDirectory != "" {
 		reportingOptions.MarkdownExporter = &markdown.Options{
-			Directory:         options.MarkdownExportDirectory,
-			IncludeRawPayload: !options.OmitRawRequests,
-			SortMode:          options.MarkdownExportSortMode,
+			Directory: options.MarkdownExportDirectory,
+			OmitRaw:   options.OmitRawRequests,
+			SortMode:  options.MarkdownExportSortMode,
 		}
 	}
 	if options.SarifExport != "" {
 		reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
 	}
 	if options.JSONExport != "" {
 		reportingOptions.JSONExporter = &jsonexporter.Options{
-			File:              options.JSONExport,
-			IncludeRawPayload: !options.OmitRawRequests,
+			File:    options.JSONExport,
+			OmitRaw: options.OmitRawRequests,
 		}
 	}
 	if options.JSONLExport != "" {
 		reportingOptions.JSONLExporter = &jsonl.Options{
-			File:              options.JSONLExport,
-			IncludeRawPayload: !options.OmitRawRequests,
+			File:    options.JSONLExport,
+			OmitRaw: options.OmitRawRequests,
 		}
 	}
 
+	reportingOptions.OmitRaw = options.OmitRawRequests
 	return reportingOptions, nil
 }
 
diff --git a/pkg/reporting/exporters/jsonexporter/jsonexporter.go b/pkg/reporting/exporters/jsonexporter/jsonexporter.go
@@ -2,10 +2,11 @@ package jsonexporter
 
 import (
 	"encoding/json"
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"os"
 	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 )
 
 type Exporter struct {
@@ -17,8 +18,8 @@ type Exporter struct {
 // Options contains the configuration options for JSON exporter client
 type Options struct {
 	// File is the file to export found JSON result to
-	File              string `yaml:"file"`
-	IncludeRawPayload bool   `yaml:"include-raw-payload"`
+	File    string `yaml:"file"`
+	OmitRaw bool   `yaml:"omit-raw"`
 }
 
 // New creates a new JSON exporter integration client based on options.
@@ -37,11 +38,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	exporter.mutex.Lock()
 	defer exporter.mutex.Unlock()
 
-	// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
-	// writing them to the list of events.
-	// This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since
-	// the property is set to "omitempty"
-	if !exporter.options.IncludeRawPayload {
+	if exporter.options.OmitRaw {
 		event.Request = ""
 		event.Response = ""
 	}
diff --git a/pkg/reporting/exporters/jsonl/jsonl.go b/pkg/reporting/exporters/jsonl/jsonl.go
@@ -2,10 +2,11 @@ package jsonl
 
 import (
 	"encoding/json"
-	"github.com/pkg/errors"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"os"
 	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 )
 
 type Exporter struct {
@@ -17,8 +18,8 @@ type Exporter struct {
 // Options contains the configuration options for JSONL exporter client
 type Options struct {
 	// File is the file to export found JSONL result to
-	File              string `yaml:"file"`
-	IncludeRawPayload bool   `yaml:"include-raw-payload"`
+	File    string `yaml:"file"`
+	OmitRaw bool   `yaml:"omit-raw"`
 }
 
 // New creates a new JSONL exporter integration client based on options.
@@ -37,11 +38,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	exporter.mutex.Lock()
 	defer exporter.mutex.Unlock()
 
-	// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
-	// writing them to the list of events.
-	// This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since
-	// the property is set to "omitempty"
-	if !exporter.options.IncludeRawPayload {
+	if exporter.options.OmitRaw {
 		event.Request = ""
 		event.Response = ""
 	}
diff --git a/pkg/reporting/exporters/markdown/markdown.go b/pkg/reporting/exporters/markdown/markdown.go
@@ -26,9 +26,9 @@ type Exporter struct {
 // Options contains the configuration options for GitHub issue tracker client
 type Options struct {
 	// Directory is the directory to export found results to
-	Directory         string `yaml:"directory"`
-	IncludeRawPayload bool   `yaml:"include-raw-payload"`
-	SortMode          string `yaml:"sort-mode"`
+	Directory string `yaml:"directory"`
+	OmitRaw   bool   `yaml:"omit-raw"`
+	SortMode  string `yaml:"sort-mode"`
 }
 
 // New creates a new markdown exporter integration client based on options.
@@ -56,15 +56,6 @@ func New(options *Options) (*Exporter, error) {
 
 // Export exports a passed result event to markdown
 func (exporter *Exporter) Export(event *output.ResultEvent) error {
-	// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
-	// writing them to the list of events.
-	// This will reduce the amount of storage as well as the fields being excluded from the markdown report output since
-	// the property is set to "omitempty"
-	if !exporter.options.IncludeRawPayload {
-		event.Request = ""
-		event.Response = ""
-	}
-
 	// index file generation
 	file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644)
 	if err != nil {
@@ -114,7 +105,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	dataBuilder.WriteString(util.CreateHeading3(format.Summary(event)))
 	dataBuilder.WriteString("\n")
 	dataBuilder.WriteString(util.CreateHorizontalLine())
-	dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{}))
+	dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{}, exporter.options.OmitRaw))
 	data := dataBuilder.Bytes()
 
 	return os.WriteFile(filepath.Join(exporter.directory, subdirectory, filename), data, 0644)
diff --git a/pkg/reporting/format/format_utils.go b/pkg/reporting/format/format_utils.go
@@ -34,7 +34,7 @@ func GetMatchedTemplateName(event *output.ResultEvent) string {
 	return matchedTemplateName
 }
 
-func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter) string {
+func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter, omitRaw bool) string {
 	template := GetMatchedTemplateName(event)
 	builder := &bytes.Buffer{}
 	builder.WriteString(fmt.Sprintf("%s: %s matched at %s\n\n", formatter.MakeBold("Details"), formatter.MakeBold(template), event.Host))
@@ -51,20 +51,22 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte
 	builder.WriteString("\n\n")
 	builder.WriteString(CreateTemplateInfoTable(&event.Info, formatter))
 
-	if event.Request != "" {
-		builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http"))
-	}
-	if event.Response != "" {
-		var responseString string
-		// If the response is larger than 5 kb, truncate it before writing.
-		maxKbSize := 5 * 1024
-		if len(event.Response) > maxKbSize {
-			responseString = event.Response[:maxKbSize]
-			responseString += ".... Truncated ...."
-		} else {
-			responseString = event.Response
+	if !omitRaw {
+		if event.Request != "" {
+			builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http"))
+		}
+		if event.Response != "" {
+			var responseString string
+			// If the response is larger than 5 kb, truncate it before writing.
+			maxKbSize := 5 * 1024
+			if len(event.Response) > maxKbSize {
+				responseString = event.Response[:maxKbSize]
+				responseString += ".... Truncated ...."
+			} else {
+				responseString = event.Response
+			}
+			builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http"))
 		}
-		builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http"))
 	}
 
 	if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go
@@ -39,4 +39,5 @@ type Options struct {
 	JSONLExporter *jsonl.Options `yaml:"jsonl"`
 
 	HttpClient *retryablehttp.Client `yaml:"-"`
+	OmitRaw    bool                  `yaml:"-"`
 }
diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go
@@ -99,6 +99,7 @@ func New(options *Options, db string) (Client, error) {
 
 	if options.GitHub != nil {
 		options.GitHub.HttpClient = options.HttpClient
+		options.GitHub.OmitRaw = options.OmitRaw
 		tracker, err := github.New(options.GitHub)
 		if err != nil {
 			return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)
@@ -107,6 +108,7 @@ func New(options *Options, db string) (Client, error) {
 	}
 	if options.GitLab != nil {
 		options.GitLab.HttpClient = options.HttpClient
+		options.GitLab.OmitRaw = options.OmitRaw
 		tracker, err := gitlab.New(options.GitLab)
 		if err != nil {
 			return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)
@@ -115,6 +117,7 @@ func New(options *Options, db string) (Client, error) {
 	}
 	if options.Jira != nil {
 		options.Jira.HttpClient = options.HttpClient
+		options.Jira.OmitRaw = options.OmitRaw
 		tracker, err := jira.New(options.Jira)
 		if err != nil {
 			return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)
diff --git a/pkg/reporting/trackers/github/github.go b/pkg/reporting/trackers/github/github.go
@@ -3,6 +3,11 @@ package github
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
 	"github.com/google/go-github/github"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -11,10 +16,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
 	"golang.org/x/oauth2"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
 )
 
 // Integration is a client for an issue tracker integration
@@ -45,6 +46,7 @@ type Options struct {
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check"`
 
 	HttpClient *retryablehttp.Client `yaml:"-"`
+	OmitRaw    bool                  `yaml:"-"`
 }
 
 // New creates a new issue tracker integration client based on options.
@@ -80,7 +82,7 @@ func New(options *Options) (*Integration, error) {
 // CreateIssue creates an issue in the tracker
 func (i *Integration) CreateIssue(event *output.ResultEvent) (err error) {
 	summary := format.Summary(event)
-	description := format.CreateReportDescription(event, util.MarkdownFormatter{})
+	description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)
 	labels := []string{}
 	severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
 	if i.options.SeverityAsLabel && severityLabel != "" {
diff --git a/pkg/reporting/trackers/gitlab/gitlab.go b/pkg/reporting/trackers/gitlab/gitlab.go
@@ -37,6 +37,7 @@ type Options struct {
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
 
 	HttpClient *retryablehttp.Client `yaml:"-"`
+	OmitRaw    bool                  `yaml:"-"`
 }
 
 // New creates a new issue tracker integration client based on options.
@@ -62,7 +63,7 @@ func New(options *Options) (*Integration, error) {
 // CreateIssue creates an issue in the tracker
 func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 	summary := format.Summary(event)
-	description := format.CreateReportDescription(event, util.MarkdownFormatter{})
+	description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)
 	labels := []string{}
 	severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
 	if i.options.SeverityAsLabel && severityLabel != "" {
diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go
@@ -77,6 +77,7 @@ type Options struct {
 	// that will be used to create the issue
 	CustomFields map[string]interface{} `yaml:"custom-fields" json:"custom_fields"`
 	StatusNot    string                 `yaml:"status-not" json:"status_not"`
+	OmitRaw      bool                   `yaml:"-"`
 }
 
 // New creates a new issue tracker integration client based on options.
@@ -154,7 +155,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
 		}
 	}
 	fields := &jira.IssueFields{
-		Description: format.CreateReportDescription(event, i),
+		Description: format.CreateReportDescription(event, i, i.options.OmitRaw),
 		Unknowns:    customFields,
 		Type:        jira.IssueType{Name: i.options.IssueType},
 		Project:     jira.Project{Key: i.options.ProjectName},
@@ -164,7 +165,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
 	if !i.options.Cloud {
 		fields = &jira.IssueFields{
 			Assignee:    &jira.User{Name: i.options.AccountID},
-			Description: format.CreateReportDescription(event, i),
+			Description: format.CreateReportDescription(event, i, i.options.OmitRaw),
 			Type:        jira.IssueType{Name: i.options.IssueType},
 			Project:     jira.Project{Key: i.options.ProjectName},
 			Summary:     summary,
@@ -196,7 +197,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 			return err
 		} else if issueID != "" {
 			_, _, err = i.jira.Issue.AddComment(issueID, &jira.Comment{
-				Body: format.CreateReportDescription(event, i),
+				Body: format.CreateReportDescription(event, i, i.options.OmitRaw),
 			})
 			return err
 		}

Original file line number	Diff line number	Diff line change
`@@ -262,27 +262,28 @@ func createReportingOptions(options types.Options) (reporting.Options, error)`
`262`	`262`	`}`
`263`	`263`	`if options.MarkdownExportDirectory != "" {`
`264`	`264`	`reportingOptions.MarkdownExporter = &markdown.Options{`
`265`		`- Directory: options.MarkdownExportDirectory,`
`266`		`- IncludeRawPayload: !options.OmitRawRequests,`
`267`		`- SortMode: options.MarkdownExportSortMode,`
	`265`	`+ Directory: options.MarkdownExportDirectory,`
	`266`	`+ OmitRaw: options.OmitRawRequests,`
	`267`	`+ SortMode: options.MarkdownExportSortMode,`
`268`	`268`	`}`
`269`	`269`	`}`
`270`	`270`	`if options.SarifExport != "" {`
`271`	`271`	`reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}`
`272`	`272`	`}`
`273`	`273`	`if options.JSONExport != "" {`
`274`	`274`	`reportingOptions.JSONExporter = &jsonexporter.Options{`
`275`		`- File: options.JSONExport,`
`276`		`- IncludeRawPayload: !options.OmitRawRequests,`
	`275`	`+ File: options.JSONExport,`
	`276`	`+ OmitRaw: options.OmitRawRequests,`
`277`	`277`	`}`
`278`	`278`	`}`
`279`	`279`	`if options.JSONLExport != "" {`
`280`	`280`	`reportingOptions.JSONLExporter = &jsonl.Options{`
`281`		`- File: options.JSONLExport,`
`282`		`- IncludeRawPayload: !options.OmitRawRequests,`
	`281`	`+ File: options.JSONLExport,`
	`282`	`+ OmitRaw: options.OmitRawRequests,`
`283`	`283`	`}`
`284`	`284`	`}`
`285`	`285`
	`286`	`+ reportingOptions.OmitRaw = options.OmitRawRequests`
`286`	`287`	`return reportingOptions, nil`
`287`	`288`	`}`
`288`	`289`
Original file line number	Diff line number	Diff line change
`@@ -39,4 +39,5 @@ type Options struct {`
`39`	`39`	JSONLExporter *jsonl.Options `yaml:"jsonl"`
`40`	`40`
`41`	`41`	HttpClient *retryablehttp.Client `yaml:"-"`
	`42`	+ OmitRaw bool `yaml:"-"`
`42`	`43`	`}`