Merge pull request #6364 from projectdiscovery/loading-performance-improvements-v2

feat: loading templates performance improvements

Author: Ice3man543

internal/runner/options.go

@@ -39,6 +39,8 @@ const (
 	DefaultDumpTrafficOutputFolder = "output"
 )
 
+var validateOptions = validator.New()
+
 func ConfigureOptions() error {
 	// with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions
 	// if file has the extension `.yaml` or `.json` we consider those as strings and not files to be read
@@ -138,8 +140,7 @@ func ParseOptions(options *types.Options) {
 
 // validateOptions validates the configuration options passed
 func ValidateOptions(options *types.Options) error {
-	validate := validator.New()
-	if err := validate.Struct(options); err != nil {
+	if err := validateOptions.Struct(options); err != nil {
 		if _, ok := err.(*validator.InvalidValidationError); ok {
 			return err
 		}

pkg/operators/matchers/validate.go

@@ -8,22 +8,29 @@ import (
 
 	"github.com/antchfx/xpath"
 	sliceutil "github.com/projectdiscovery/utils/slice"
-	"gopkg.in/yaml.v3"
 )
 
 var commonExpectedFields = []string{"Type", "Condition", "Name", "MatchAll", "Negative", "Internal"}
 
 // Validate perform initial validation on the matcher structure
 func (matcher *Matcher) Validate() error {
-	// uses yaml marshaling to convert the struct to map[string]interface to have same field names
+	// Build a map of YAML‐tag names that are actually set (non-zero) in the matcher.
 	matcherMap := make(map[string]interface{})
-	marshaledMatcher, err := yaml.Marshal(matcher)
-	if err != nil {
-		return err
-	}
-	if err := yaml.Unmarshal(marshaledMatcher, &matcherMap); err != nil {
-		return err
+	val := reflect.ValueOf(*matcher)
+	typ := reflect.TypeOf(*matcher)
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		// skip internal / unexported or opt-out fields
+		yamlTag := strings.Split(field.Tag.Get("yaml"), ",")[0]
+		if yamlTag == "" || yamlTag == "-" {
+			continue
+		}
+		if val.Field(i).IsZero() {
+			continue
+		}
+		matcherMap[yamlTag] = struct{}{}
 	}
+	var err error
 
 	var expectedFields []string
 	switch matcher.matcherType {

pkg/templates/parser.go

@@ -131,13 +131,13 @@ func (p *Parser) ParseTemplate(templatePath string, catalog catalog.Catalog) (an
 		_ = reader.Close()
 	}()
 
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, err
-	}
-
-	// pre-process directives only for local files
+	// For local YAML files, check if preprocessing is needed
+	var data []byte
 	if fileutil.FileExists(templatePath) && config.GetTemplateFormatFromExt(templatePath) == config.YAML {
+		data, err = io.ReadAll(reader)
+		if err != nil {
+			return nil, err
+		}
 		data, err = yamlutil.PreProcess(data)
 		if err != nil {
 			return nil, err
@@ -148,12 +148,28 @@ func (p *Parser) ParseTemplate(templatePath string, catalog catalog.Catalog) (an
 
 	switch config.GetTemplateFormatFromExt(templatePath) {
 	case config.JSON:
+		if data == nil {
+			data, err = io.ReadAll(reader)
+			if err != nil {
+				return nil, err
+			}
+		}
 		err = json.Unmarshal(data, template)
 	case config.YAML:
-		if p.NoStrictSyntax {
-			err = yaml.Unmarshal(data, template)
+		if data != nil {
+			// Already read and preprocessed
+			if p.NoStrictSyntax {
+				err = yaml.Unmarshal(data, template)
+			} else {
+				err = yaml.UnmarshalStrict(data, template)
+			}
 		} else {
-			err = yaml.UnmarshalStrict(data, template)
+			// Stream directly from reader
+			decoder := yaml.NewDecoder(reader)
+			if !p.NoStrictSyntax {
+				decoder.SetStrict(true)
+			}
+			err = decoder.Decode(template)
 		}
 	default:
 		err = fmt.Errorf("failed to identify template format expected JSON or YAML but got %v", templatePath)
@@ -162,7 +178,7 @@ func (p *Parser) ParseTemplate(templatePath string, catalog catalog.Catalog) (an
 		return nil, err
 	}
 
-	p.parsedTemplatesCache.Store(templatePath, template, data, nil)
+	p.parsedTemplatesCache.Store(templatePath, template, nil, nil) // don't keep raw bytes to save memory
 	return template, nil
 }
 

pkg/templates/templates.go

@@ -7,7 +7,6 @@ import (
 	"strconv"
 	"strings"
 
-	validate "github.com/go-playground/validator/v10"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/code"
@@ -310,10 +309,8 @@ func (template *Template) validateAllRequestIDs() {
 // MarshalYAML forces recursive struct validation during marshal operation
 func (template *Template) MarshalYAML() ([]byte, error) {
 	out, marshalErr := yaml.Marshal(template)
-	// Review: we are adding requestIDs for templateContext
-	// if we are using this method then we might need to purge manually added IDS that start with `templatetype_`
-	// this is only applicable if there are more than 1 request fields in protocol
-	errValidate := validate.New().Struct(template)
+	// Use shared validator to avoid rebuilding struct cache for every template marshal
+	errValidate := tplValidator.Struct(template)
 	return out, multierr.Append(marshalErr, errValidate)
 }
 
@@ -354,7 +351,7 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
 	if len(alias.RequestsWithTCP) > 0 {
 		template.RequestsNetwork = alias.RequestsWithTCP
 	}
-	err = validate.New().Struct(template)
+	err = tplValidator.Struct(template)
 	if err != nil {
 		return err
 	}
@@ -525,7 +522,7 @@ func (template *Template) hasMultipleRequests() bool {
 func (template *Template) MarshalJSON() ([]byte, error) {
 	type TemplateAlias Template //avoid recursion
 	out, marshalErr := json.Marshal((*TemplateAlias)(template))
-	errValidate := validate.New().Struct(template)
+	errValidate := tplValidator.Struct(template)
 	return out, multierr.Append(marshalErr, errValidate)
 }
 
@@ -538,7 +535,7 @@ func (template *Template) UnmarshalJSON(data []byte) error {
 		return err
 	}
 	*template = Template(*alias)
-	err = validate.New().Struct(template)
+	err = tplValidator.Struct(template)
 	if err != nil {
 		return err
 	}

pkg/templates/validator_singleton.go

@@ -0,0 +1,7 @@
+package templates
+
+import (
+	validate "github.com/go-playground/validator/v10"
+)
+
+var tplValidator = validate.New()