Match/filter

README.md

@@ -65,6 +65,11 @@ SOURCE:
    -all                            Use all sources (slow) for enumeration
    -es, -exclude-sources string[]  sources to exclude from enumeration (-es archiveis,zoomeye)
 
+FILTER:
+   -m, -match string[]   subdomain or list of subdomain to match (file or comma separated)
+   -f, -filter string[]   subdomain or list of subdomain to filter (file or comma separated)
+
+
 RATE-LIMIT:
    -rl, -rate-limit int  maximum number of http requests to send per second
    -t int                number of concurrent goroutines for resolving (-active only) (default 10)

v2/go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.18 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.19 // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect
 	github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc // indirect
 	github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect

v2/go.sum

@@ -13,7 +13,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
-github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -168,8 +168,9 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe
 github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
-github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
 github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
+github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c=
+github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE=
 github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -211,8 +212,8 @@ github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d/go.mod
 github.com/projectdiscovery/chaos-client v0.2.0 h1:a6B0HGAUdlpU4YR0o7j70GaY/PkEdtrP9VsRacfEeng=
 github.com/projectdiscovery/chaos-client v0.2.0/go.mod h1:6A43jvxrzIS93GZXUlrHfqr8jDZO4g/QCr6At1h9r2U=
 github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
-github.com/projectdiscovery/dnsx v1.1.0 h1:Wjtx4BJQBemIk+Jaibf6bmisPHgovXfX+V1iH+pJ2YU=
-github.com/projectdiscovery/dnsx v1.1.0/go.mod h1:TtcjQcFABuFIN7C+jOQ00PX41UnXp6aU3V9/O1hc0YE=
+github.com/projectdiscovery/dnsx v1.1.0 h1:nHyAI4+1HXevex0ilADtOmhvx7BxctCVmbj91YaV3Vo=
+github.com/projectdiscovery/dnsx v1.1.0/go.mod h1:gWtUhnqycbb/g+We7O977ZXDN0bZZf3twZBZcv9R61I=
 github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg=
 github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI=
 github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
@@ -228,7 +229,7 @@ github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLY
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/hmap v0.0.2-0.20220308104318-929a45d8e775/go.mod h1:RWJgRqpBPJW1R9CzcEH3zmiqPFEOfcbn6lLvHD9xrKE=
+github.com/projectdiscovery/hmap v0.0.2/go.mod h1:YU3TeNTDmLW2dtb4LvuEtDsPTsQ06XMnmOeD3KOuU6c=
 github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc h1:C0L6pUvVI+sPJSBaPQJEG/HjPtg8Mgs2vEpsdrl064A=
 github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
 github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=

v2/pkg/runner/enumerate.go

@@ -18,7 +18,6 @@ const maxNumCount = 2
 // EnumerateSingleDomain performs subdomain enumeration against a single domain
 func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outputs []io.Writer) error {
 	gologger.Info().Msgf("Enumerating subdomains for %s\n", domain)
-
 	// Get the API keys for sources from the configuration
 	// and also create the active resolving engine for the domain.
 	keys := r.options.Providers.GetKeys()
@@ -58,32 +57,33 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 				}
 				subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "")
 
-				if _, ok := uniqueMap[subdomain]; !ok {
-					sourceMap[subdomain] = make(map[string]struct{})
-				}
-
-				// Log the verbose message about the found subdomain per source
-				if _, ok := sourceMap[subdomain][result.Source]; !ok {
-					gologger.Verbose().Label(result.Source).Msg(subdomain)
-				}
-
-				sourceMap[subdomain][result.Source] = struct{}{}
-
-				// Check if the subdomain is a duplicate. If not,
-				// send the subdomain for resolution.
-				if _, ok := uniqueMap[subdomain]; ok {
-					continue
-				}
-
-				hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source}
-
-				uniqueMap[subdomain] = hostEntry
-
-				// If the user asked to remove wildcard then send on the resolve
-				// queue. Otherwise, if mode is not verbose print the results on
-				// the screen as they are discovered.
-				if r.options.RemoveWildcard {
-					resolutionPool.Tasks <- hostEntry
+				if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain {
+					if _, ok := uniqueMap[subdomain]; !ok {
+						sourceMap[subdomain] = make(map[string]struct{})
+					}
+
+					// Log the verbose message about the found subdomain per source
+					if _, ok := sourceMap[subdomain][result.Source]; !ok {
+						gologger.Verbose().Label(result.Source).Msg(subdomain)
+					}
+
+					sourceMap[subdomain][result.Source] = struct{}{}
+
+					// Check if the subdomain is a duplicate. If not,
+					// send the subdomain for resolution.
+					if _, ok := uniqueMap[subdomain]; ok {
+						continue
+					}
+
+					hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source}
+
+					uniqueMap[subdomain] = hostEntry
+					// If the user asked to remove wildcard then send on the resolve
+					// queue. Otherwise, if mode is not verbose print the results on
+					// the screen as they are discovered.
+					if r.options.RemoveWildcard {
+						resolutionPool.Tasks <- hostEntry
+					}
 				}
 			}
 		}
@@ -112,9 +112,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 		}
 	}
 	wg.Wait()
-
 	outputter := NewOutputter(r.options.JSON)
-
 	// Now output all results in output writers
 	var err error
 	for _, w := range outputs {
@@ -144,6 +142,24 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 	} else {
 		gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
 	}
-
 	return nil
 }
+
+func (r *Runner) filterAndMatchSubdomain(subdomain string) bool {
+	if r.options.filterRegexes != nil {
+		for _, filter := range r.options.filterRegexes {
+			if m := filter.MatchString(subdomain); m {
+				return false
+			}
+		}
+	}
+	if r.options.matchRegexes != nil {
+		for _, match := range r.options.matchRegexes {
+			if m := match.MatchString(subdomain); m {
+				return true
+			}
+		}
+		return false
+	}
+	return true
+}

v2/pkg/runner/options.go

@@ -9,6 +9,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"strings"
 	"time"
 
@@ -62,8 +63,12 @@ type Options struct {
 	Proxy          string              // HTTP proxy
 	RateLimit      int                 // Maximum number of HTTP requests to send per second
 	// YAMLConfig contains the unmarshalled yaml config file
-	Providers  *Providers
-	ExcludeIps bool
+	Providers     *Providers
+	ExcludeIps    bool
+	Match         goflags.StringSlice
+	Filter        goflags.StringSlice
+	matchRegexes  []*regexp.Regexp
+	filterRegexes []*regexp.Regexp
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -101,6 +106,11 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", []string{}, "sources to exclude from enumeration (-es archiveis,zoomeye)", goflags.NormalizedStringSliceOptions),
 	)
 
+	createGroup(flagSet, "filter", "Filter",
+		flagSet.StringSliceVarP(&options.Match, "match", "m", []string{}, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Filter, "filter", "f", []string{}, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions),
+	)
+
 	createGroup(flagSet, "rate-limit", "Rate-limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"),
 		flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"),

v2/pkg/runner/validate.go

@@ -2,6 +2,8 @@ package runner
 
 import (
 	"errors"
+	"regexp"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
@@ -34,8 +36,31 @@ func (options *Options) validateOptions() error {
 		return errors.New("hostip flag must be used with RemoveWildcard option")
 	}
 
+	if options.Match != nil {
+		options.matchRegexes = make([]*regexp.Regexp, len(options.Match))
+		var err error
+		for i, re := range options.Match {
+			if options.matchRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil {
+				return errors.New("invalid value for match regex option")
+			}
+		}
+	}
+	if options.Filter != nil {
+		options.filterRegexes = make([]*regexp.Regexp, len(options.Filter))
+		var err error
+		for i, re := range options.Filter {
+			if options.filterRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil {
+				return errors.New("invalid value for filter regex option")
+			}
+		}
+	}
 	return nil
 }
+func stripRegexString(val string) string {
+	val = strings.ReplaceAll(val, ".", "\\.")
+	val = strings.ReplaceAll(val, "*", ".*")
+	return val
+}
 
 // configureOutput configures the output on the screen
 func (options *Options) configureOutput() {