Merge remote-tracking branch 'origin'

Author: ehsandeep

README.md

@@ -137,6 +137,7 @@ TEMPLATES:
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all available templates
+   -tgl                                   list all available tags
    -sign                                  signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
    -code                                  enable loading code protocol-based templates
    -dut, -disable-unsigned-templates      disable running unsigned templates or templates with mismatched signature

cmd/integration-test/integration-test.go

@@ -37,6 +37,7 @@ var (
 		"dns":             dnsTestCases,
 		"workflow":        workflowTestcases,
 		"loader":          loaderTestcases,
+		"profile-loader":  profileLoaderTestcases,
 		"websocket":       websocketTestCases,
 		"headless":        headlessTestcases,
 		"whois":           whoisTestCases,

cmd/integration-test/profile-loader.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	errorutil "github.com/projectdiscovery/utils/errors"
+)
+
+var profileLoaderTestcases = []TestCaseInfo{
+	{Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}},
+	{Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}},
+	{Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}},
+}
+
+type profileLoaderByRelFile struct{}
+
+func (h *profileLoaderByRelFile) Execute(testName string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "kev.yml")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) < 267 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 267, len(results))
+	}
+	return nil
+}
+
+type profileLoaderById struct{}
+
+func (h *profileLoaderById) Execute(testName string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "kev")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) < 267 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 267, len(results))
+	}
+	return nil
+}
+
+type customProfileLoader struct{}
+
+func (h *customProfileLoader) Execute(filepath string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", filepath)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) < 267 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 267, len(results))
+	}
+	return nil
+}

cmd/nuclei/main.go

@@ -42,9 +42,10 @@ import (
 )
 
 var (
-	cfgFile    string
-	memProfile string // optional profile file path
-	options    = &types.Options{}
+	cfgFile         string
+	templateProfile string
+	memProfile      string // optional profile file path
+	options         = &types.Options{}
 )
 
 func main() {
@@ -225,6 +226,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+		flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"),
 		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"),
 		flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
 		flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),
@@ -270,6 +272,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("configs", "Configurations",
 		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&templateProfile, "profile", "tp", "", "template profile config file to run"),
+		flagSet.BoolVarP(&options.ListTemplateProfiles, "profile-list", "tpl", false, "list community template profiles"),
 		flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"),
 		flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
 		flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
@@ -497,6 +501,34 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
 	}
 
+	defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
+	if templateProfile != "" {
+		if filepath.Ext(templateProfile) == "" {
+			if tp := findProfilePathById(templateProfile, defaultProfilesPath); tp != "" {
+				templateProfile = tp
+			} else {
+				gologger.Fatal().Msgf("'%s' is not a profile-id or profile path", templateProfile)
+			}
+		}
+		if !filepath.IsAbs(templateProfile) {
+			if filepath.Dir(templateProfile) == "profiles" {
+				defaultProfilesPath = filepath.Join(config.DefaultConfig.GetTemplateDir())
+			}
+			currentDir, err := os.Getwd()
+			if err == nil && fileutil.FileExists(filepath.Join(currentDir, templateProfile)) {
+				templateProfile = filepath.Join(currentDir, templateProfile)
+			} else {
+				templateProfile = filepath.Join(defaultProfilesPath, templateProfile)
+			}
+		}
+		if !fileutil.FileExists(templateProfile) {
+			gologger.Fatal().Msgf("given template profile file '%s' does not exist", templateProfile)
+		}
+		if err := flagSet.MergeConfigFile(templateProfile); err != nil {
+			gologger.Fatal().Msgf("Could not read template profile: %s\n", err)
+		}
+	}
+
 	if len(options.SecretsFile) > 0 {
 		for _, secretFile := range options.SecretsFile {
 			if !fileutil.FileExists(secretFile) {
@@ -622,6 +654,27 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 	os.Exit(0)
 }
 
+func findProfilePathById(profileId, templatesDir string) string {
+	var profilePath string
+	err := filepath.WalkDir(templatesDir, func(iterItem string, d fs.DirEntry, err error) error {
+		ext := filepath.Ext(iterItem)
+		isYaml := ext == extensions.YAML || ext == extensions.YML
+		if err != nil || d.IsDir() || !isYaml {
+			// skip non yaml files
+			return nil
+		}
+		if strings.TrimSuffix(filepath.Base(iterItem), ext) == profileId {
+			profilePath = iterItem
+			return fmt.Errorf("FOUND")
+		}
+		return nil
+	})
+	if err != nil && err.Error() != "FOUND" {
+		gologger.Error().Msgf("%s\n", err)
+	}
+	return profilePath
+}
+
 func init() {
 	// print stacktrace of errors in debug mode
 	if strings.EqualFold(os.Getenv("DEBUG"), "true") {

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.68
+	github.com/projectdiscovery/fastdialer v0.0.69
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.45
+	github.com/projectdiscovery/rawhttp v0.1.47
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.57
+	github.com/projectdiscovery/retryablehttp-go v1.0.58
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -81,24 +81,25 @@ require (
 	github.com/projectdiscovery/dsl v0.0.52
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.49
+	github.com/projectdiscovery/goflags v0.1.50
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
 	github.com/projectdiscovery/httpx v1.6.0
 	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.38
+	github.com/projectdiscovery/ratelimit v0.0.39
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/useragent v0.0.47
-	github.com/projectdiscovery/utils v0.0.91
-	github.com/projectdiscovery/wappalyzergo v0.0.116
+	github.com/projectdiscovery/useragent v0.0.48
+	github.com/projectdiscovery/utils v0.0.92
+	github.com/projectdiscovery/wappalyzergo v0.0.120
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
+	github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
 	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -142,8 +143,6 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/eapache/channels v1.1.0 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -179,6 +178,7 @@ require (
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -232,6 +232,7 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	github.com/zcalusic/sysinfo v1.0.2 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect