diff --git a/internal/runner/runner.go b/internal/runner/runner.go index f7954c2ce7..d72d986036 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -459,6 +459,9 @@ func (r *Runner) RunEnumeration() error { disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent) templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose) + // purge global caches primarily used for loading templates + config.DefaultConfig.PurgeGlobalCache() + // add the hosts from the metadata queries of loaded templates into input provider if r.options.Uncover && len(r.options.UncoverQuery) == 0 { uncoverOpts := &uncoverlib.Options{ diff --git a/lib/multi.go b/lib/multi.go index 6dcee59dde..e09ea40bf2 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -5,6 +5,7 @@ import ( "time" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" "github.com/projectdiscovery/nuclei/v3/pkg/core/inputs" @@ -88,6 +89,7 @@ func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error { // GlobalResultCallback sets a callback function which will be called for each result func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) { e.eng.resultCallbacks = []func(*output.ResultEvent){callback} + config.DefaultConfig.PurgeGlobalCache() } // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index 5ff7e8c5d9..bb9008b5e2 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -45,6 +45,9 @@ type Config struct { LatestNucleiTemplatesVersion string `json:"nuclei-templates-latest-version"` LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` + // Other AppLevel/Global Settings + registerdCaches []GlobalCache `json:"-"` // registered global caches + // internal / unexported fields disableUpdates bool `json:"-"` // disable updates both version check and template updates homeDir string `json:"-"` // User Home Directory @@ -298,6 +301,19 @@ func (c *Config) WriteTemplatesIndex(index map[string]string) error { return os.WriteFile(indexFile, buff.Bytes(), 0600) } +// RegisterGlobalCache registers a global cache at app level +// and is available to be purged on demand +func (c *Config) RegisterGlobalCache(cache GlobalCache) { + c.registerdCaches = append(c.registerdCaches, cache) +} + +// PurgeGlobalCache purges all registered global caches +func (c *Config) PurgeGlobalCache() { + for _, cache := range c.registerdCaches { + cache.Purge() + } +} + // getTemplatesConfigFilePath returns configDir/.templates-config.json file path func (c *Config) getTemplatesConfigFilePath() string { return filepath.Join(c.configDir, TemplateConfigFileName) diff --git a/pkg/catalog/config/template.go b/pkg/catalog/config/template.go index 2b7ea83ed0..ff78ef09eb 100644 --- a/pkg/catalog/config/template.go +++ b/pkg/catalog/config/template.go @@ -13,6 +13,13 @@ import ( stringsutil "github.com/projectdiscovery/utils/strings" ) +// GlobalCache are global cache that have global +// scope and are not purged but can be purged +// via config.DefaultConfig +type GlobalCache interface { + Purge() +} + var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"} // TemplateFormat diff --git a/pkg/parsers/parser.go b/pkg/parsers/parser.go index 01bb3d3fdf..46e1fe046c 100644 --- a/pkg/parsers/parser.go +++ b/pkg/parsers/parser.go @@ -150,6 +150,7 @@ const ( func init() { parsedTemplatesCache = cache.New() + config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache) stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") diff --git a/pkg/templates/cache/cache.go b/pkg/templates/cache/cache.go index 65e7d37f79..2b06a758a4 100644 --- a/pkg/templates/cache/cache.go +++ b/pkg/templates/cache/cache.go @@ -1,17 +1,17 @@ package cache import ( - "sync" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Templates is a cache for caching and storing templates for reuse. type Templates struct { - items *sync.Map + items *mapsutil.SyncLockMap[string, parsedTemplateErrHolder] } // New returns a new templates cache func New() *Templates { - return &Templates{items: &sync.Map{}} + return &Templates{items: mapsutil.NewSyncLockMap[string, parsedTemplateErrHolder]()} } type parsedTemplateErrHolder struct { @@ -22,18 +22,19 @@ type parsedTemplateErrHolder struct { // Has returns true if the cache has a template. The template // is returned along with any errors if found. func (t *Templates) Has(template string) (interface{}, error) { - value, ok := t.items.Load(template) - if !ok || value == nil { - return nil, nil - } - templateError, ok := value.(parsedTemplateErrHolder) + value, ok := t.items.Get(template) if !ok { return nil, nil } - return templateError.template, templateError.err + return value.template, value.err } // Store stores a template with data and error func (t *Templates) Store(template string, data interface{}, err error) { - t.items.Store(template, parsedTemplateErrHolder{template: data, err: err}) + _ = t.items.Set(template, parsedTemplateErrHolder{template: data, err: err}) +} + +// Purge the cache +func (t *Templates) Purge() { + t.items.Clear() } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 3bffe7b9f5..67b4d4ea6a 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -45,6 +45,7 @@ func init() { SignatureStats[verifier.Identifier()] = &atomic.Uint64{} } SignatureStats[Unsigned] = &atomic.Uint64{} + config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache) } // Parse parses a yaml request template file