diff --git a/internal/runner/options.go b/internal/runner/options.go index 828957c3cb..76346c927a 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -262,9 +262,9 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } if options.MarkdownExportDirectory != "" { reportingOptions.MarkdownExporter = &markdown.Options{ - Directory: options.MarkdownExportDirectory, - IncludeRawPayload: !options.OmitRawRequests, - SortMode: options.MarkdownExportSortMode, + Directory: options.MarkdownExportDirectory, + OmitRaw: options.OmitRawRequests, + SortMode: options.MarkdownExportSortMode, } } if options.SarifExport != "" { @@ -272,17 +272,18 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } if options.JSONExport != "" { reportingOptions.JSONExporter = &jsonexporter.Options{ - File: options.JSONExport, - IncludeRawPayload: !options.OmitRawRequests, + File: options.JSONExport, + OmitRaw: options.OmitRawRequests, } } if options.JSONLExport != "" { reportingOptions.JSONLExporter = &jsonl.Options{ - File: options.JSONLExport, - IncludeRawPayload: !options.OmitRawRequests, + File: options.JSONLExport, + OmitRaw: options.OmitRawRequests, } } + reportingOptions.OmitRaw = options.OmitRawRequests return reportingOptions, nil } diff --git a/pkg/reporting/exporters/jsonexporter/jsonexporter.go b/pkg/reporting/exporters/jsonexporter/jsonexporter.go index 6881fc4b35..c540cb714c 100644 --- a/pkg/reporting/exporters/jsonexporter/jsonexporter.go +++ b/pkg/reporting/exporters/jsonexporter/jsonexporter.go @@ -2,10 +2,11 @@ package jsonexporter import ( "encoding/json" - "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v3/pkg/output" "os" "sync" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v3/pkg/output" ) type Exporter struct { @@ -17,8 +18,8 @@ type Exporter struct { // Options contains the configuration options for JSON exporter client type Options struct { // File is the file to export found JSON result to - File string `yaml:"file"` - IncludeRawPayload bool `yaml:"include-raw-payload"` + File string `yaml:"file"` + OmitRaw bool `yaml:"omit-raw"` } // New creates a new JSON exporter integration client based on options. @@ -37,11 +38,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { exporter.mutex.Lock() defer exporter.mutex.Unlock() - // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid - // writing them to the list of events. - // This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since - // the property is set to "omitempty" - if !exporter.options.IncludeRawPayload { + if exporter.options.OmitRaw { event.Request = "" event.Response = "" } diff --git a/pkg/reporting/exporters/jsonl/jsonl.go b/pkg/reporting/exporters/jsonl/jsonl.go index 5a55950325..c02f90baba 100644 --- a/pkg/reporting/exporters/jsonl/jsonl.go +++ b/pkg/reporting/exporters/jsonl/jsonl.go @@ -2,10 +2,11 @@ package jsonl import ( "encoding/json" - "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v3/pkg/output" "os" "sync" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v3/pkg/output" ) type Exporter struct { @@ -17,8 +18,8 @@ type Exporter struct { // Options contains the configuration options for JSONL exporter client type Options struct { // File is the file to export found JSONL result to - File string `yaml:"file"` - IncludeRawPayload bool `yaml:"include-raw-payload"` + File string `yaml:"file"` + OmitRaw bool `yaml:"omit-raw"` } // New creates a new JSONL exporter integration client based on options. @@ -37,11 +38,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { exporter.mutex.Lock() defer exporter.mutex.Unlock() - // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid - // writing them to the list of events. - // This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since - // the property is set to "omitempty" - if !exporter.options.IncludeRawPayload { + if exporter.options.OmitRaw { event.Request = "" event.Response = "" } diff --git a/pkg/reporting/exporters/markdown/markdown.go b/pkg/reporting/exporters/markdown/markdown.go index 462978b2f1..c56d94c752 100644 --- a/pkg/reporting/exporters/markdown/markdown.go +++ b/pkg/reporting/exporters/markdown/markdown.go @@ -26,9 +26,9 @@ type Exporter struct { // Options contains the configuration options for GitHub issue tracker client type Options struct { // Directory is the directory to export found results to - Directory string `yaml:"directory"` - IncludeRawPayload bool `yaml:"include-raw-payload"` - SortMode string `yaml:"sort-mode"` + Directory string `yaml:"directory"` + OmitRaw bool `yaml:"omit-raw"` + SortMode string `yaml:"sort-mode"` } // New creates a new markdown exporter integration client based on options. @@ -56,15 +56,6 @@ func New(options *Options) (*Exporter, error) { // Export exports a passed result event to markdown func (exporter *Exporter) Export(event *output.ResultEvent) error { - // If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid - // writing them to the list of events. - // This will reduce the amount of storage as well as the fields being excluded from the markdown report output since - // the property is set to "omitempty" - if !exporter.options.IncludeRawPayload { - event.Request = "" - event.Response = "" - } - // index file generation file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644) if err != nil { @@ -114,7 +105,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error { dataBuilder.WriteString(util.CreateHeading3(format.Summary(event))) dataBuilder.WriteString("\n") dataBuilder.WriteString(util.CreateHorizontalLine()) - dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{})) + dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{}, exporter.options.OmitRaw)) data := dataBuilder.Bytes() return os.WriteFile(filepath.Join(exporter.directory, subdirectory, filename), data, 0644) diff --git a/pkg/reporting/format/format_utils.go b/pkg/reporting/format/format_utils.go index 286ebcc3f9..253742b37d 100644 --- a/pkg/reporting/format/format_utils.go +++ b/pkg/reporting/format/format_utils.go @@ -34,7 +34,7 @@ func GetMatchedTemplateName(event *output.ResultEvent) string { return matchedTemplateName } -func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter) string { +func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter, omitRaw bool) string { template := GetMatchedTemplateName(event) builder := &bytes.Buffer{} builder.WriteString(fmt.Sprintf("%s: %s matched at %s\n\n", formatter.MakeBold("Details"), formatter.MakeBold(template), event.Host)) @@ -51,20 +51,22 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte builder.WriteString("\n\n") builder.WriteString(CreateTemplateInfoTable(&event.Info, formatter)) - if event.Request != "" { - builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http")) - } - if event.Response != "" { - var responseString string - // If the response is larger than 5 kb, truncate it before writing. - maxKbSize := 5 * 1024 - if len(event.Response) > maxKbSize { - responseString = event.Response[:maxKbSize] - responseString += ".... Truncated ...." - } else { - responseString = event.Response + if !omitRaw { + if event.Request != "" { + builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http")) + } + if event.Response != "" { + var responseString string + // If the response is larger than 5 kb, truncate it before writing. + maxKbSize := 5 * 1024 + if len(event.Response) > maxKbSize { + responseString = event.Response[:maxKbSize] + responseString += ".... Truncated ...." + } else { + responseString = event.Response + } + builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http")) } - builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http")) } if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go index 110db38f48..19762c8662 100644 --- a/pkg/reporting/options.go +++ b/pkg/reporting/options.go @@ -39,4 +39,5 @@ type Options struct { JSONLExporter *jsonl.Options `yaml:"jsonl"` HttpClient *retryablehttp.Client `yaml:"-"` + OmitRaw bool `yaml:"-"` } diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index cd6f5cc06d..9902493fc6 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -99,6 +99,7 @@ func New(options *Options, db string) (Client, error) { if options.GitHub != nil { options.GitHub.HttpClient = options.HttpClient + options.GitHub.OmitRaw = options.OmitRaw tracker, err := github.New(options.GitHub) if err != nil { return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) @@ -107,6 +108,7 @@ func New(options *Options, db string) (Client, error) { } if options.GitLab != nil { options.GitLab.HttpClient = options.HttpClient + options.GitLab.OmitRaw = options.OmitRaw tracker, err := gitlab.New(options.GitLab) if err != nil { return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) @@ -115,6 +117,7 @@ func New(options *Options, db string) (Client, error) { } if options.Jira != nil { options.Jira.HttpClient = options.HttpClient + options.Jira.OmitRaw = options.OmitRaw tracker, err := jira.New(options.Jira) if err != nil { return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) diff --git a/pkg/reporting/trackers/github/github.go b/pkg/reporting/trackers/github/github.go index dab374f55d..f099fb873b 100644 --- a/pkg/reporting/trackers/github/github.go +++ b/pkg/reporting/trackers/github/github.go @@ -3,6 +3,11 @@ package github import ( "context" "fmt" + "io" + "net/http" + "net/url" + "strings" + "github.com/google/go-github/github" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/output" @@ -11,10 +16,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryablehttp-go" "golang.org/x/oauth2" - "io" - "net/http" - "net/url" - "strings" ) // Integration is a client for an issue tracker integration @@ -45,6 +46,7 @@ type Options struct { DuplicateIssueCheck bool `yaml:"duplicate-issue-check"` HttpClient *retryablehttp.Client `yaml:"-"` + OmitRaw bool `yaml:"-"` } // New creates a new issue tracker integration client based on options. @@ -80,7 +82,7 @@ func New(options *Options) (*Integration, error) { // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) (err error) { summary := format.Summary(event) - description := format.CreateReportDescription(event, util.MarkdownFormatter{}) + description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw) labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) if i.options.SeverityAsLabel && severityLabel != "" { diff --git a/pkg/reporting/trackers/gitlab/gitlab.go b/pkg/reporting/trackers/gitlab/gitlab.go index 630ffc7465..22c191f576 100644 --- a/pkg/reporting/trackers/gitlab/gitlab.go +++ b/pkg/reporting/trackers/gitlab/gitlab.go @@ -37,6 +37,7 @@ type Options struct { DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"` HttpClient *retryablehttp.Client `yaml:"-"` + OmitRaw bool `yaml:"-"` } // New creates a new issue tracker integration client based on options. @@ -62,7 +63,7 @@ func New(options *Options) (*Integration, error) { // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) - description := format.CreateReportDescription(event, util.MarkdownFormatter{}) + description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw) labels := []string{} severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) if i.options.SeverityAsLabel && severityLabel != "" { diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go index b68d48d2b4..77302f5add 100644 --- a/pkg/reporting/trackers/jira/jira.go +++ b/pkg/reporting/trackers/jira/jira.go @@ -77,6 +77,7 @@ type Options struct { // that will be used to create the issue CustomFields map[string]interface{} `yaml:"custom-fields" json:"custom_fields"` StatusNot string `yaml:"status-not" json:"status_not"` + OmitRaw bool `yaml:"-"` } // New creates a new issue tracker integration client based on options. @@ -154,7 +155,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { } } fields := &jira.IssueFields{ - Description: format.CreateReportDescription(event, i), + Description: format.CreateReportDescription(event, i, i.options.OmitRaw), Unknowns: customFields, Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, @@ -164,7 +165,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { if !i.options.Cloud { fields = &jira.IssueFields{ Assignee: &jira.User{Name: i.options.AccountID}, - Description: format.CreateReportDescription(event, i), + Description: format.CreateReportDescription(event, i, i.options.OmitRaw), Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, @@ -196,7 +197,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error { return err } else if issueID != "" { _, _, err = i.jira.Issue.AddComment(issueID, &jira.Comment{ - Body: format.CreateReportDescription(event, i), + Body: format.CreateReportDescription(event, i, i.options.OmitRaw), }) return err }