From 032dcd2223abc68e7e23535440fbcc0d5f4208f2 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Mar 2024 00:01:38 +0100 Subject: [PATCH 1/6] purge cache on global callback set --- go.mod | 2 +- pkg/templates/cache/cache.go | 21 +++++++++++---------- pkg/templates/compile.go | 5 +++++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 4fc625edf8..20eb279521 100644 --- a/go.mod +++ b/go.mod @@ -320,7 +320,7 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/labstack/gommon v0.4.0 // indirect + github.com/labstack/gommon v0.4.0 github.com/mattn/go-colorable v0.1.13 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect diff --git a/pkg/templates/cache/cache.go b/pkg/templates/cache/cache.go index 65e7d37f79..8d06beed99 100644 --- a/pkg/templates/cache/cache.go +++ b/pkg/templates/cache/cache.go @@ -1,17 +1,17 @@ package cache import ( - "sync" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Templates is a cache for caching and storing templates for reuse. type Templates struct { - items *sync.Map + items *mapsutil.SyncLockMap[string, parsedTemplateErrHolder] } // New returns a new templates cache func New() *Templates { - return &Templates{items: &sync.Map{}} + return &Templates{items: mapsutil.NewSyncLockMap[string, parsedTemplateErrHolder]()} } type parsedTemplateErrHolder struct { @@ -22,18 +22,19 @@ type parsedTemplateErrHolder struct { // Has returns true if the cache has a template. The template // is returned along with any errors if found. func (t *Templates) Has(template string) (interface{}, error) { - value, ok := t.items.Load(template) - if !ok || value == nil { - return nil, nil - } - templateError, ok := value.(parsedTemplateErrHolder) + value, ok := t.items.Get(template) if !ok { return nil, nil } - return templateError.template, templateError.err + return value.template, value.err } // Store stores a template with data and error func (t *Templates) Store(template string, data interface{}, err error) { - t.items.Store(template, parsedTemplateErrHolder{template: data, err: err}) + t.items.Set(template, parsedTemplateErrHolder{template: data, err: err}) +} + +// Purge the cache +func (t *Templates) Purge() { + t.items.Clear() } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 3bffe7b9f5..5ce2090164 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -47,6 +47,11 @@ func init() { SignatureStats[Unsigned] = &atomic.Uint64{} } +// Purge the cache +func PurgeCache() { + parsedTemplatesCache.Purge() +} + // Parse parses a yaml request template file // TODO make sure reading from the disk the template parsing happens once: see parsers.ParseTemplate vs templates.Parse // From e485b3382379b56fe1f5dcfa438c99ded44f0536 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Mar 2024 00:09:13 +0100 Subject: [PATCH 2/6] lint --- go.mod | 2 +- pkg/templates/cache/cache.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 20eb279521..4fc625edf8 100644 --- a/go.mod +++ b/go.mod @@ -320,7 +320,7 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/labstack/gommon v0.4.0 + github.com/labstack/gommon v0.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect diff --git a/pkg/templates/cache/cache.go b/pkg/templates/cache/cache.go index 8d06beed99..2b06a758a4 100644 --- a/pkg/templates/cache/cache.go +++ b/pkg/templates/cache/cache.go @@ -31,7 +31,7 @@ func (t *Templates) Has(template string) (interface{}, error) { // Store stores a template with data and error func (t *Templates) Store(template string, data interface{}, err error) { - t.items.Set(template, parsedTemplateErrHolder{template: data, err: err}) + _ = t.items.Set(template, parsedTemplateErrHolder{template: data, err: err}) } // Purge the cache From e73896b32719b3a2d389078c720e3b04351a3fc1 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 9 Mar 2024 22:06:35 +0100 Subject: [PATCH 3/6] purging cache --- lib/multi.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/multi.go b/lib/multi.go index 6dcee59dde..44fd590fb0 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/parsers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/ratelimit" errorutil "github.com/projectdiscovery/utils/errors" @@ -88,6 +89,7 @@ func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error { // GlobalResultCallback sets a callback function which will be called for each result func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) { e.eng.resultCallbacks = []func(*output.ResultEvent){callback} + templates.PurgeCache() } // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) From f2cc78a0fe8e584b2f50c3372a19e32291c4fd66 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 11 Mar 2024 01:02:26 +0530 Subject: [PATCH 4/6] purge cache in runner after loading templates --- internal/runner/runner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index f7954c2ce7..47923a07ad 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -459,6 +459,9 @@ func (r *Runner) RunEnumeration() error { disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent) templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose) + // purge cache once everything is loaded + templates.PurgeCache() + // add the hosts from the metadata queries of loaded templates into input provider if r.options.Uncover && len(r.options.UncoverQuery) == 0 { uncoverOpts := &uncoverlib.Options{ From 6ca34aecd7ad592e9d088a9a75c6e8f619e4ecbb Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 11 Mar 2024 01:17:20 +0530 Subject: [PATCH 5/6] include internal cache from parsers + add global cache register/purge via config --- internal/runner/runner.go | 4 ++-- lib/multi.go | 4 ++-- pkg/catalog/config/nucleiconfig.go | 22 ++++++++++++++++++++++ pkg/catalog/config/template.go | 7 +++++++ pkg/parsers/parser.go | 1 + pkg/templates/compile.go | 6 +----- 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 47923a07ad..d72d986036 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -459,8 +459,8 @@ func (r *Runner) RunEnumeration() error { disk.PrintDeprecatedPathsMsgIfApplicable(r.options.Silent) templates.PrintDeprecatedProtocolNameMsgIfApplicable(r.options.Silent, r.options.Verbose) - // purge cache once everything is loaded - templates.PurgeCache() + // purge global caches primarily used for loading templates + config.DefaultConfig.PurgeGlobalCache() // add the hosts from the metadata queries of loaded templates into input provider if r.options.Uncover && len(r.options.UncoverQuery) == 0 { diff --git a/lib/multi.go b/lib/multi.go index 44fd590fb0..e09ea40bf2 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -5,6 +5,7 @@ import ( "time" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" "github.com/projectdiscovery/nuclei/v3/pkg/core/inputs" @@ -12,7 +13,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/parsers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/ratelimit" errorutil "github.com/projectdiscovery/utils/errors" @@ -89,7 +89,7 @@ func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error { // GlobalResultCallback sets a callback function which will be called for each result func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *output.ResultEvent)) { e.eng.resultCallbacks = []func(*output.ResultEvent){callback} - templates.PurgeCache() + config.DefaultConfig.PurgeGlobalCache() } // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index 5ff7e8c5d9..44a2d69f47 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -45,6 +45,10 @@ type Config struct { LatestNucleiTemplatesVersion string `json:"nuclei-templates-latest-version"` LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` + // Other AppLevel/Global Settings + DisableGlobalCachePurge bool `json:"-"` // when enabled disables purging of global cache(useful for app as service) + registerdCaches []GlobalCache `json:"-"` // registered global caches + // internal / unexported fields disableUpdates bool `json:"-"` // disable updates both version check and template updates homeDir string `json:"-"` // User Home Directory @@ -298,6 +302,24 @@ func (c *Config) WriteTemplatesIndex(index map[string]string) error { return os.WriteFile(indexFile, buff.Bytes(), 0600) } +// RegisterGlobalCache registers a global cache at app level +// and is available to be purged on demand +func (c *Config) RegisterGlobalCache(cache GlobalCache) { + c.registerdCaches = append(c.registerdCaches, cache) +} + +// PurgeGlobalCache purges all registered global caches +func (c *Config) PurgeGlobalCache() { + if c.DisableGlobalCachePurge { + // useful for apps running nuclei as background process + // where we don't want to purge global cache + return + } + for _, cache := range c.registerdCaches { + cache.Purge() + } +} + // getTemplatesConfigFilePath returns configDir/.templates-config.json file path func (c *Config) getTemplatesConfigFilePath() string { return filepath.Join(c.configDir, TemplateConfigFileName) diff --git a/pkg/catalog/config/template.go b/pkg/catalog/config/template.go index 2b7ea83ed0..ff78ef09eb 100644 --- a/pkg/catalog/config/template.go +++ b/pkg/catalog/config/template.go @@ -13,6 +13,13 @@ import ( stringsutil "github.com/projectdiscovery/utils/strings" ) +// GlobalCache are global cache that have global +// scope and are not purged but can be purged +// via config.DefaultConfig +type GlobalCache interface { + Purge() +} + var knownConfigFiles = []string{"cves.json", "contributors.json", "TEMPLATES-STATS.json"} // TemplateFormat diff --git a/pkg/parsers/parser.go b/pkg/parsers/parser.go index 01bb3d3fdf..46e1fe046c 100644 --- a/pkg/parsers/parser.go +++ b/pkg/parsers/parser.go @@ -150,6 +150,7 @@ const ( func init() { parsedTemplatesCache = cache.New() + config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache) stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 5ce2090164..67b4d4ea6a 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -45,11 +45,7 @@ func init() { SignatureStats[verifier.Identifier()] = &atomic.Uint64{} } SignatureStats[Unsigned] = &atomic.Uint64{} -} - -// Purge the cache -func PurgeCache() { - parsedTemplatesCache.Purge() + config.DefaultConfig.RegisterGlobalCache(parsedTemplatesCache) } // Parse parses a yaml request template file From 05d44bd4ecf4264d18afd8a6c738b03a9dadb0a0 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 11 Mar 2024 01:20:49 +0530 Subject: [PATCH 6/6] remove disable cache purge option --- pkg/catalog/config/nucleiconfig.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index 44a2d69f47..bb9008b5e2 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -46,8 +46,7 @@ type Config struct { LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` // Other AppLevel/Global Settings - DisableGlobalCachePurge bool `json:"-"` // when enabled disables purging of global cache(useful for app as service) - registerdCaches []GlobalCache `json:"-"` // registered global caches + registerdCaches []GlobalCache `json:"-"` // registered global caches // internal / unexported fields disableUpdates bool `json:"-"` // disable updates both version check and template updates @@ -310,11 +309,6 @@ func (c *Config) RegisterGlobalCache(cache GlobalCache) { // PurgeGlobalCache purges all registered global caches func (c *Config) PurgeGlobalCache() { - if c.DisableGlobalCachePurge { - // useful for apps running nuclei as background process - // where we don't want to purge global cache - return - } for _, cache := range c.registerdCaches { cache.Purge() }