filter out near duplicate responses

runner/options.go

@@ -187,6 +187,7 @@ type Options struct {
 	OutputMatchContentLength  string
 	OutputFilterStatusCode    string
 	OutputFilterErrorPage     bool
+	FilterOutDuplicates       bool
 	OutputFilterContentLength string
 	InputRawRequest           string
 	rawRequest                string
@@ -409,6 +410,7 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("filters", "Filters",
 		flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
 		flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"),
+		flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses"),
 		flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
 		flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
 		flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),

runner/runner.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corona10/goimagehash"
+	"github.com/mfonda/simhash"
 	asnmap "github.com/projectdiscovery/asnmap/libs"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/httpx/common/customextract"
@@ -86,6 +87,7 @@ type Runner struct {
 	browser             *Browser
 	errorPageClassifier *errorpageclassifier.ErrorPageClassifier
 	pHashClusters       []pHashCluster
+	simHashes           map[uint64]struct{}
 	httpApiEndpoint     *Server
 }
 
@@ -359,6 +361,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	runner.errorPageClassifier = errorpageclassifier.New()
+	runner.simHashes = make(map[uint64]struct{})
 
 	if options.HttpApiEndpoint != "" {
 		apiServer := NewServer(options.HttpApiEndpoint, options)
@@ -514,6 +517,23 @@ func (r *Runner) seen(k string) bool {
 	return ok
 }
 
+func (r *Runner) duplicate(resp []byte) bool {
+	respSimHash := simhash.Simhash(simhash.NewWordFeatureSet(resp))
+	if _, exists := r.simHashes[respSimHash]; exists {
+		gologger.Warning().Msgf("Skipping duplicate response with simhash %d\n", respSimHash)
+		return true
+	}
+	for simHash := range r.simHashes {
+		// lower threshold for increased precision
+		if simhash.Compare(simHash, respSimHash) <= 3 {
+			gologger.Warning().Msgf("Skipping near-duplicate response with simhash %d\n", respSimHash)
+			return true
+		}
+	}
+	r.simHashes[respSimHash] = struct{}{}
+	return false
+}
+
 func (r *Runner) testAndSet(k string) bool {
 	// skip empty lines
 	k = strings.TrimSpace(k)
@@ -884,6 +904,11 @@ func (r *Runner) RunEnumeration() {
 				logFilteredErrorPage(r.options.OutputFilterErrorPagePath, resp.URL)
 				continue
 			}
+
+			if r.options.FilterOutDuplicates && r.duplicate(resp.Response.Data) {
+				continue
+			}
+
 			if len(r.options.filterStatusCode) > 0 && sliceutil.Contains(r.options.filterStatusCode, resp.StatusCode) {
 				continue
 			}