Match/filter

README.md

@@ -65,6 +65,11 @@ SOURCE:
    -all                            Use all sources (slow) for enumeration
    -es, -exclude-sources string[]  sources to exclude from enumeration (-es archiveis,zoomeye)
 
+FILTER:
+   -m, -match string[]   subdomain or list of subdomain to match (file or comma separated)
+   -f, -filter string[]   subdomain or list of subdomain to filter (file or comma separated)
+
+
 RATE-LIMIT:
    -rl, -rate-limit int  maximum number of http requests to send per second
    -t int                number of concurrent goroutines for resolving (-active only) (default 10)

v2/go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.18 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.19 // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect
 	github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc // indirect
 	github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect

v2/go.sum

@@ -168,8 +168,9 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe
 github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
-github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
 github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
+github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c=
+github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE=
 github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

v2/pkg/runner/config_test.go

@@ -1,10 +1,9 @@
 package runner
 
 import (
+	"github.com/stretchr/testify/require"
 	"os"
 	"testing"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func TestConfigGetDirectory(t *testing.T) {
@@ -18,5 +17,5 @@ func TestConfigGetDirectory(t *testing.T) {
 	}
 	config := home + "/.config/subfinder"
 
-	assert.Equal(t, directory, config, "Directory and config should be equal")
+	require.Equal(t, directory, config, "Directory and config should be equal")
 }

v2/pkg/runner/enumerate.go

@@ -18,7 +18,6 @@ const maxNumCount = 2
 // EnumerateSingleDomain performs subdomain enumeration against a single domain
 func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outputs []io.Writer) error {
 	gologger.Info().Msgf("Enumerating subdomains for %s\n", domain)
-
 	// Get the API keys for sources from the configuration
 	// and also create the active resolving engine for the domain.
 	keys := r.options.Providers.GetKeys()
@@ -58,32 +57,33 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 				}
 				subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "")
 
-				if _, ok := uniqueMap[subdomain]; !ok {
-					sourceMap[subdomain] = make(map[string]struct{})
-				}
-
-				// Log the verbose message about the found subdomain per source
-				if _, ok := sourceMap[subdomain][result.Source]; !ok {
-					gologger.Verbose().Label(result.Source).Msg(subdomain)
-				}
-
-				sourceMap[subdomain][result.Source] = struct{}{}
-
-				// Check if the subdomain is a duplicate. If not,
-				// send the subdomain for resolution.
-				if _, ok := uniqueMap[subdomain]; ok {
-					continue
-				}
-
-				hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source}
-
-				uniqueMap[subdomain] = hostEntry
-
-				// If the user asked to remove wildcard then send on the resolve
-				// queue. Otherwise, if mode is not verbose print the results on
-				// the screen as they are discovered.
-				if r.options.RemoveWildcard {
-					resolutionPool.Tasks <- hostEntry
+				if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain {
+					if _, ok := uniqueMap[subdomain]; !ok {
+						sourceMap[subdomain] = make(map[string]struct{})
+					}
+
+					// Log the verbose message about the found subdomain per source
+					if _, ok := sourceMap[subdomain][result.Source]; !ok {
+						gologger.Verbose().Label(result.Source).Msg(subdomain)
+					}
+
+					sourceMap[subdomain][result.Source] = struct{}{}
+
+					// Check if the subdomain is a duplicate. If not,
+					// send the subdomain for resolution.
+					if _, ok := uniqueMap[subdomain]; ok {
+						continue
+					}
+
+					hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source}
+
+					uniqueMap[subdomain] = hostEntry
+					// If the user asked to remove wildcard then send on the resolve
+					// queue. Otherwise, if mode is not verbose print the results on
+					// the screen as they are discovered.
+					if r.options.RemoveWildcard {
+						resolutionPool.Tasks <- hostEntry
+					}
 				}
 			}
 		}
@@ -112,9 +112,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 		}
 	}
 	wg.Wait()
-
 	outputter := NewOutputter(r.options.JSON)
-
 	// Now output all results in output writers
 	var err error
 	for _, w := range outputs {
@@ -144,6 +142,24 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 	} else {
 		gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
 	}
-
 	return nil
 }
+
+func (r *Runner) filterAndMatchSubdomain(subdomain string) bool {
+	if r.options.filterRegexes != nil {
+		for _, filter := range r.options.filterRegexes {
+			if m := filter.MatchString(subdomain); m {
+				return false
+			}
+		}
+	}
+	if r.options.matchRegexes != nil {
+		for _, match := range r.options.matchRegexes {
+			if m := match.MatchString(subdomain); m {
+				return true
+			}
+		}
+		return false
+	}
+	return true
+}

v2/pkg/runner/enumerate_test.go

@@ -0,0 +1,145 @@
+package runner
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilterAndMatchSubdomain(t *testing.T) {
+	options := &Options{}
+	options.Domain = []string{"example.com"}
+	options.Threads = 10
+	options.Timeout = 10
+	options.Output = os.Stdout
+	t.Run("Literal Match", func(t *testing.T) {
+		options.Match = []string{"req.example.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		match := runner.filterAndMatchSubdomain("req.example.com")
+		require.True(t, match, "Expecting a boolean True value ")
+	})
+	t.Run("Multiple Wildcards Match", func(t *testing.T) {
+		options.Match = []string{"*.ns.*.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"}
+		for _, sub := range subdomain {
+			match := runner.filterAndMatchSubdomain(sub)
+			require.True(t, match, "Expecting a boolean True value ")
+		}
+	})
+	t.Run("Sequential Match", func(t *testing.T) {
+		options.Match = []string{"*.ns.example.com", "*.hackerone.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		subdomain := []string{"a.ns.example.com", "b.hackerone.com"}
+		for _, sub := range subdomain {
+			match := runner.filterAndMatchSubdomain(sub)
+			require.True(t, match, "Expecting a boolean True value ")
+		}
+	})
+	t.Run("Literal Filter", func(t *testing.T) {
+		options.Filter = []string{"req.example.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		match := runner.filterAndMatchSubdomain("req.example.com")
+		require.False(t, match, "Expecting a boolean False value ")
+	})
+	t.Run("Multiple Wildcards Filter", func(t *testing.T) {
+		options.Filter = []string{"*.ns.*.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"}
+		for _, sub := range subdomain {
+			match := runner.filterAndMatchSubdomain(sub)
+			require.False(t, match, "Expecting a boolean False value ")
+		}
+	})
+	t.Run("Sequential Filter", func(t *testing.T) {
+		options.Filter = []string{"*.ns.example.com", "*.hackerone.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		subdomain := []string{"a.ns.example.com", "b.hackerone.com"}
+		for _, sub := range subdomain {
+			match := runner.filterAndMatchSubdomain(sub)
+			require.False(t, match, "Expecting a boolean False value ")
+		}
+	})
+	t.Run("Filter and Match", func(t *testing.T) {
+		options.Filter = []string{"example.com"}
+		options.Match = []string{"hackerone.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		subdomain := []string{"example.com", "example.com"}
+		for _, sub := range subdomain {
+			match := runner.filterAndMatchSubdomain(sub)
+			require.False(t, match, "Expecting a boolean False value ")
+		}
+	})
+
+	t.Run("Filter and Match - Same Root Domain", func(t *testing.T) {
+		options.Filter = []string{"example.com"}
+		options.Match = []string{"www.example.com"}
+		err := options.validateOptions()
+		if err != nil {
+			t.Fatalf("Expected nil got %v while validation\n", err)
+		}
+		runner, err := NewRunner(options)
+		if err != nil {
+			t.Fatalf("Expected nil got %v while creating runner\n", err)
+		}
+		subdomain := map[string]string{"filter": "example.com", "match": "www.example.com"}
+		for key, sub := range subdomain {
+			result := runner.filterAndMatchSubdomain(sub)
+			if key == "filter" {
+				require.False(t, result, "Expecting a boolean False value ")
+			} else {
+				require.True(t, result, "Expecting a boolean True value ")
+			}
+		}
+	})
+}

v2/pkg/runner/options.go

@@ -9,6 +9,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"strings"
 	"time"
 
@@ -62,8 +63,12 @@ type Options struct {
 	Proxy          string              // HTTP proxy
 	RateLimit      int                 // Maximum number of HTTP requests to send per second
 	// YAMLConfig contains the unmarshalled yaml config file
-	Providers  *Providers
-	ExcludeIps bool
+	Providers     *Providers
+	ExcludeIps    bool
+	Match         goflags.StringSlice
+	Filter        goflags.StringSlice
+	matchRegexes  []*regexp.Regexp
+	filterRegexes []*regexp.Regexp
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -101,6 +106,11 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", []string{}, "sources to exclude from enumeration (-es archiveis,zoomeye)", goflags.NormalizedStringSliceOptions),
 	)
 
+	createGroup(flagSet, "filter", "Filter",
+		flagSet.StringSliceVarP(&options.Match, "match", "m", []string{}, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Filter, "filter", "f", []string{}, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions),
+	)
+
 	createGroup(flagSet, "rate-limit", "Rate-limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"),
 		flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"),

v2/pkg/runner/validate.go

@@ -2,6 +2,9 @@ package runner
 
 import (
 	"errors"
+	"fmt"
+	"regexp"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
@@ -34,8 +37,31 @@ func (options *Options) validateOptions() error {
 		return errors.New("hostip flag must be used with RemoveWildcard option")
 	}
 
+	if options.Match != nil {
+		options.matchRegexes = make([]*regexp.Regexp, len(options.Match))
+		var err error
+		for i, re := range options.Match {
+			if options.matchRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil {
+				return errors.New("invalid value for match regex option")
+			}
+		}
+	}
+	if options.Filter != nil {
+		options.filterRegexes = make([]*regexp.Regexp, len(options.Filter))
+		var err error
+		for i, re := range options.Filter {
+			if options.filterRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil {
+				return errors.New("invalid value for filter regex option")
+			}
+		}
+	}
 	return nil
 }
+func stripRegexString(val string) string {
+	val = strings.ReplaceAll(val, ".", "\\.")
+	val = strings.ReplaceAll(val, "*", ".*")
+	return fmt.Sprint("^", val, "$")
+}
 
 // configureOutput configures the output on the screen
 func (options *Options) configureOutput() {