Skip to content

Commit f034407

Browse files
Devang-SolankiehsandeeptarunKoyalwar
authored
Added netlas.io (#829)
* added netlas to sources * added netlas to sources * added netlas to sources * tried to solve lint errors * Improved Readability * fix nil panics * netlas: use session to send http request * update dev version * Revert "update dev version" This reverts commit e29ab3e. * make provider-config error non fatal --------- Co-authored-by: Sandeep Singh <[email protected]> Co-authored-by: Tarun Koyalwar <[email protected]>
1 parent 5ac9d83 commit f034407

File tree

4 files changed

+199
-2
lines changed

4 files changed

+199
-2
lines changed

‎v2/pkg/passive/sources.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter"
3232
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
3333
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
34+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
3435
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal"
3536
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
3637
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
@@ -70,6 +71,7 @@ var AllSources = [...]subscraping.Source{
7071
&hackertarget.Source{},
7172
&hunter.Source{},
7273
&intelx.Source{},
74+
&netlas.Source{},
7375
&leakix.Source{},
7476
&passivetotal.Source{},
7577
&quake.Source{},

‎v2/pkg/passive/sources_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ var (
3232
"github",
3333
"hackertarget",
3434
"intelx",
35+
"netlas",
3536
"passivetotal",
3637
"quake",
3738
"rapiddns",

‎v2/pkg/runner/options.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,8 +222,8 @@ func (options *Options) loadProvidersFrom(location string) {
222222

223223
// We skip bailing out if file doesn't exist because we'll create it
224224
// at the end of options parsing from default via goflags.
225-
if err := UnmarshalFrom(location); isFatalErr(err) && !errors.Is(err, os.ErrNotExist) {
226-
gologger.Fatal().Msgf("Could not read providers from %s: %s\n", location, err)
225+
if err := UnmarshalFrom(location); err != nil && (!strings.Contains(err.Error(), "file doesn't exist") || errors.Is(os.ErrNotExist, err)) {
226+
gologger.Error().Msgf("Could not read providers from %s: %s\n", location, err)
227227
}
228228
}
229229

‎v2/pkg/subscraping/sources/netlas/netlas.go‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
// Package netlas logic
2+
package netlas
3+
4+
import (
5+
"context"
6+
"io"
7+
8+
"encoding/json"
9+
"fmt"
10+
"net/http"
11+
"net/url"
12+
"strconv"
13+
"time"
14+
15+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
16+
)
17+
18+
type Item struct {
19+
Data struct {
20+
A []string `json:"a,omitempty"`
21+
Txt []string `json:"txt,omitempty"`
22+
LastUpdated string `json:"last_updated,omitempty"`
23+
Timestamp string `json:"@timestamp,omitempty"`
24+
Ns []string `json:"ns,omitempty"`
25+
Level int `json:"level,omitempty"`
26+
Zone string `json:"zone,omitempty"`
27+
Domain string `json:"domain,omitempty"`
28+
Cname []string `json:"cname,omitempty"`
29+
Mx []string `json:"mx,omitempty"`
30+
} `json:"data"`
31+
}
32+
33+
type DomainsResponse struct {
34+
Items []Item `json:"items"`
35+
Took int `json:"took"`
36+
}
37+
38+
type DomainsCountResponse struct {
39+
Count int `json:"count"`
40+
}
41+
42+
// Source is the passive scraping agent
43+
type Source struct {
44+
apiKeys []string
45+
timeTaken time.Duration
46+
errors int
47+
results int
48+
skipped bool
49+
}
50+
51+
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
52+
results := make(chan subscraping.Result)
53+
s.errors = 0
54+
s.results = 0
55+
56+
go func() {
57+
defer func(startTime time.Time) {
58+
s.timeTaken = time.Since(startTime)
59+
close(results)
60+
}(time.Now())
61+
62+
// To get count of domains
63+
endpoint := "https://app.netlas.io/api/domains_count/"
64+
params := url.Values{}
65+
countQuery := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain)
66+
params.Set("q", countQuery)
67+
countUrl := endpoint + "?" + params.Encode()
68+
69+
// Pick an API key
70+
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
71+
resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{
72+
"accept": "application/json",
73+
"X-API-Key": randomApiKey,
74+
}, nil, subscraping.BasicAuth{})
75+
76+
if err != nil {
77+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
78+
s.errors++
79+
return
80+
} else if resp.StatusCode != 200 {
81+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)}
82+
s.errors++
83+
return
84+
}
85+
defer resp.Body.Close()
86+
87+
body, err := io.ReadAll(resp.Body)
88+
if err != nil {
89+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")}
90+
s.errors++
91+
return
92+
}
93+
94+
// Parse the JSON response
95+
var domainsCount DomainsCountResponse
96+
err = json.Unmarshal(body, &domainsCount)
97+
if err != nil {
98+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
99+
s.errors++
100+
return
101+
}
102+
103+
//Define the API endpoint URL and query parameters
104+
105+
for i := 0; i < domainsCount.Count; i += 20 {
106+
107+
time.Sleep(1000 * time.Millisecond)
108+
offset := strconv.Itoa(i)
109+
110+
endpoint := "https://app.netlas.io/api/domains/"
111+
params := url.Values{}
112+
query := fmt.Sprintf("domain:(domain:*.%s AND NOT domain:%s)", domain, domain)
113+
params.Set("q", query)
114+
params.Set("source_type", "include")
115+
params.Set("start", offset)
116+
params.Set("fields", "*")
117+
apiUrl := endpoint + "?" + params.Encode()
118+
119+
// Pick an API key
120+
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
121+
122+
resp, err := session.HTTPRequest(ctx, http.MethodGet, apiUrl, "", map[string]string{
123+
"accept": "application/json",
124+
"X-API-Key": randomApiKey}, nil, subscraping.BasicAuth{})
125+
if err != nil {
126+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
127+
s.errors++
128+
return
129+
}
130+
defer resp.Body.Close()
131+
body, err := io.ReadAll(resp.Body)
132+
if err != nil {
133+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")}
134+
s.errors++
135+
return
136+
}
137+
138+
if resp.StatusCode == 429 {
139+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)}
140+
s.errors++
141+
break
142+
}
143+
144+
// Parse the response body and extract the domain values
145+
var data DomainsResponse
146+
err = json.Unmarshal(body, &data)
147+
if err != nil {
148+
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
149+
s.errors++
150+
return
151+
}
152+
153+
for _, item := range data.Items {
154+
results <- subscraping.Result{
155+
Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain,
156+
}
157+
s.results++
158+
}
159+
}
160+
161+
}()
162+
163+
return results
164+
}
165+
166+
// Name returns the name of the source
167+
func (s *Source) Name() string {
168+
return "netlas"
169+
}
170+
171+
func (s *Source) IsDefault() bool {
172+
return false
173+
}
174+
175+
func (s *Source) HasRecursiveSupport() bool {
176+
return false
177+
}
178+
179+
func (s *Source) NeedsKey() bool {
180+
return true
181+
}
182+
183+
func (s *Source) AddApiKeys(keys []string) {
184+
s.apiKeys = keys
185+
}
186+
187+
func (s *Source) Statistics() subscraping.Statistics {
188+
return subscraping.Statistics{
189+
Errors: s.errors,
190+
Results: s.results,
191+
TimeTaken: s.timeTaken,
192+
Skipped: s.skipped,
193+
}
194+
}

0 commit comments

Comments
 (0)