From 98b37d17235120a5ad28abb2b3ce426b952cfd23 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Thu, 16 Feb 2023 20:45:19 +0530 Subject: [PATCH 1/2] Update dependencies: (#761) - Bump github.com/projectdiscovery/ratelimit from v0.0.2 to v0.0.6 - Replace github.com/projectdiscovery/fileutil with github.com/projectdiscovery/utils/file - Remove &gitlab.Source{} from AllSources array --- v2/go.mod | 8 ++++---- v2/go.sum | 17 ++++++++--------- v2/pkg/passive/sources.go | 2 -- v2/pkg/passive/sources_test.go | 1 - v2/pkg/runner/options.go | 2 +- v2/pkg/runner/util.go | 2 +- 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 4fc4e166c..2881bd428 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,9 +10,9 @@ require ( github.com/projectdiscovery/chaos-client v0.3.0 github.com/projectdiscovery/dnsx v1.1.1 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b - github.com/projectdiscovery/ratelimit v0.0.2 + github.com/projectdiscovery/ratelimit v0.0.6 + github.com/projectdiscovery/utils v0.0.9 github.com/rs/xid v1.4.0 github.com/stretchr/testify v1.8.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -32,17 +32,17 @@ require ( github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect github.com/projectdiscovery/cdncheck v0.0.3 // indirect + github.com/projectdiscovery/fileutil v0.0.1 // indirect github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc // indirect github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 // indirect github.com/projectdiscovery/mapcidr v1.0.1 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 // indirect - github.com/projectdiscovery/utils v0.0.2 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/ulikunitz/xz v0.5.7 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/text v0.4.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index bb45a6162..756b9ca14 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -88,8 +88,8 @@ github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= -github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= -github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= +github.com/projectdiscovery/fileutil v0.0.1 h1:3K3UqCDOan3LsvWhV0nyvVuMWSwCloNPUJIGcXsi1os= +github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.1.5 h1:zqZvIKAdSC62e5jv5UDgBWyP61KrLHxxUXjH/MAJlB4= github.com/projectdiscovery/goflags v0.1.5/go.mod h1:mms5rLXW0+jjbFFTyl/4t9VwtP5TCE+1Y1pReezluAI= @@ -106,8 +106,8 @@ github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1: github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= -github.com/projectdiscovery/ratelimit v0.0.2 h1:vRscyCcmTWndd16Ed3lC51196eKYena+xpYfrDomjMg= -github.com/projectdiscovery/ratelimit v0.0.2/go.mod h1:WBz8N1P+CyxnfUoGfVCqah4NZ2SreSX7v9dY8wIlK70= +github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= +github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryabledns v1.0.15 h1:3Nn119UwYsfUPC3g0q57ftz0Wb5Zl5ppvw8R0Xu0DEI= github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= @@ -121,8 +121,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.2 h1:lif4OYBqd8jCf0glRBfSs2lT4nMtNjHjeysRw8HIW8M= -github.com/projectdiscovery/utils v0.0.2/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= +github.com/projectdiscovery/utils v0.0.9 h1:+bmjtRU1IWt5dKg9+9TbIaRkw/kN05myCXDOhCQRX7g= +github.com/projectdiscovery/utils v0.0.9/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= @@ -153,8 +153,9 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -197,12 +198,10 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 79bc45264..f31d6f2c5 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -27,7 +27,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/gitlab" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" @@ -67,7 +66,6 @@ var AllSources = [...]subscraping.Source{ &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, - &gitlab.Source{}, &hackertarget.Source{}, &hunter.Source{}, &intelx.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 768ec5c26..89e70b13b 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -30,7 +30,6 @@ var ( "fofa", "fullhunt", "github", - "gitlab", "hackertarget", "intelx", "passivetotal", diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 94bf63ec2..a3b13286c 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -14,7 +14,7 @@ import ( "gopkg.in/yaml.v3" - "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/utils/file" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go index c9ce3bf8f..2507e8972 100644 --- a/v2/pkg/runner/util.go +++ b/v2/pkg/runner/util.go @@ -5,7 +5,7 @@ import ( "github.com/pkg/errors" - "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/utils/file" ) var ( From 54efbfe5216ff750536082347d9c83db40f7fba3 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Fri, 17 Feb 2023 11:07:35 +0530 Subject: [PATCH 2/2] - Disable default logger - Replace hasStdin with fileutils.HasStdin --- v2/pkg/runner/options.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index a3b13286c..1b094e50c 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -15,6 +15,7 @@ import ( "gopkg.in/yaml.v3" "github.com/projectdiscovery/utils/file" + "github.com/projectdiscovery/utils/log" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" @@ -71,6 +72,7 @@ type OnResultCallback func(result *resolve.HostEntry) // ParseOptions parses the command line flags provided by a user func ParseOptions() *Options { + logutil.DisableDefaultLogger() // Seed default random number generator rand.Seed(time.Now().UnixNano()) @@ -162,7 +164,7 @@ func ParseOptions() *Options { options.Output = os.Stdout // Check if stdin pipe was given - options.Stdin = hasStdin() + options.Stdin = fileutil.HasStdin() // Read the inputs and configure the logging options.configureOutput() @@ -259,18 +261,6 @@ func isFatalErr(err error) bool { return err != nil && !errors.Is(err, io.EOF) } -func hasStdin() bool { - stat, err := os.Stdin.Stat() - if err != nil { - return false - } - - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 - isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - - return isPipedFromChrDev || isPipedFromFIFO -} - func listSources(options *Options) { gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n")