Added crt.sh CT driver

Author: lanrat

.gitignore

@@ -1,3 +1,4 @@
+Gopkg.lock
 certgraph
 build/
 *.json

Gopkg.toml

@@ -0,0 +1,4 @@
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/lib/pq"

Makefile

@@ -23,6 +23,9 @@ $(PLATFORMS): $(SOURCES)
 	CGO_ENABLED=0 GOOS=$(os) GOARCH=$(arch) go build $(BUILD_FLAGS) -o 'build/$(os)/$(arch)/certgraph$(ext)' $(SOURCES)
 	cd build/$(os)/$(arch)/; zip -r ../../certgraph-$(os)-$(arch)-$(GIT_DATE).zip .; cd ../../../
 
+dep:
+	dep ensure
+
 fmt:
 	gofmt -s -w -l .
 

certgraph.go

@@ -16,7 +16,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lanrat/certgraph/ct/google"
+	"github.com/lanrat/certgraph/driver"
+	"github.com/lanrat/certgraph/driver/crtsh"
+	"github.com/lanrat/certgraph/driver/google"
 	"github.com/lanrat/certgraph/graph"
 	"github.com/lanrat/certgraph/status"
 )
@@ -49,6 +51,8 @@ var include_ct_sub bool
 var tls_connect bool
 var ver bool
 var skipCDN bool
+var crtsh_driver bool
+var ctDriver driver.Driver
 
 func generateGraphMetadata() map[string]interface{} {
 	data := make(map[string]interface{})
@@ -81,6 +85,7 @@ func main() {
 	timeoutPtr := flag.Uint("timeout", 5, "tcp timeout in seconds")
 	flag.BoolVar(&verbose, "verbose", false, "verbose logging")
 	flag.BoolVar(&ct, "ct", false, "use certificate transparancy search to find certificates")
+	flag.BoolVar(&crtsh_driver, "crtsh", false, "use the CRT.sh api instead of Google for CT")
 	flag.BoolVar(&include_ct_sub, "ct-subdomains", false, "include sub-domains in certificate transparancy search")
 	flag.BoolVar(&skipCDN, "skip-cdn", false, "do not crawl into CDN certs")
 	flag.BoolVar(&notls, "notls", false, "don't connect to hosts to collect certificates")
@@ -117,6 +122,20 @@ func main() {
 		return
 	}
 
+	// TODO better driver support
+	if ct {
+		var err error
+		if crtsh_driver {
+			ctDriver, err = crtsh.NewCRTshDriver()
+		} else {
+			ctDriver, err = google.NewGoogleCTDriver()
+		}
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return
+		}
+	}
+
 	// set verbose loggin
 	graph.Verbose = verbose
 
@@ -268,7 +287,7 @@ func BFSVisit(node *graph.DomainNode) {
 func visitCT(node *graph.DomainNode) {
 	// perform ct search
 	// TODO do pagnation in multiple threads to not block on long searches
-	fingerprints, err := google.QueryDomain(node.Domain, false, include_ct_sub)
+	fingerprints, err := ctDriver.QueryDomain(node.Domain, false, include_ct_sub)
 	if err != nil {
 		v(err)
 		return
@@ -282,7 +301,7 @@ func visitCT(node *graph.DomainNode) {
 
 		if !exists {
 			// get cert details
-			certnode, err = google.GetCert(fp)
+			certnode, err = ctDriver.QueryCert(fp)
 			if err != nil {
 				v(err)
 				continue

ct/.gitignore

@@ -0,0 +1,95 @@
+package crtsh
+
+import (
+	"database/sql"
+	"fmt"
+	"github.com/lanrat/certgraph/driver"
+	"github.com/lanrat/certgraph/graph"
+	_ "github.com/lib/pq"
+)
+
+const connStr = "postgresql://guest@crt.sh/certwatch?sslmode=disable"
+
+type crtsh struct {
+	db *sql.DB
+}
+
+func NewCRTshDriver() (driver.Driver, error) {
+	d := new(crtsh)
+	var err error
+
+	d.db, err = sql.Open("postgres", connStr)
+
+	return d, err
+}
+
+func (d *crtsh) QueryDomain(domain string, include_expired bool, include_subdomains bool) ([]graph.Fingerprint, error) {
+	results := make([]graph.Fingerprint, 0, 5)
+
+	queryStr := "SELECT digest(certificate.certificate, 'sha256') sha256 FROM certificate_identity, certificate WHERE certificate.id = certificate_identity.certificate_id AND x509_notAfter(certificate.certificate) > statement_timestamp() AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))"
+	if include_expired {
+		queryStr = "SELECT digest(certificate.certificate, 'sha256') sha256 FROM certificate_identity, certificate WHERE certificate.id = certificate_identity.certificate_id AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))"
+	}
+
+	if include_subdomains {
+		domain = fmt.Sprintf("%%.%s", domain)
+	}
+
+	rows, err := d.db.Query(queryStr, domain)
+	if err != nil {
+		return results, err
+	}
+
+	for rows.Next() {
+		var hash []byte
+		err = rows.Scan(&hash)
+		if err != nil {
+			return results, err
+		}
+		results = append(results, graph.FingerprintFromBytes(hash))
+	}
+
+	return results, nil
+}
+
+func (d *crtsh) QueryCert(fp graph.Fingerprint) (*graph.CertNode, error) {
+	certnode := new(graph.CertNode)
+	certnode.Fingerprint = fp
+	certnode.Domains = make([]string, 0, 5)
+	certnode.CT = true
+
+	queryStr := "SELECT DISTINCT certificate_identity.name_value FROM certificate, certificate_identity WHERE certificate.id = certificate_identity.certificate_id AND digest(certificate.certificate, 'sha256') = $1"
+
+	rows, err := d.db.Query(queryStr, fp[:])
+	if err != nil {
+		return certnode, err
+	}
+
+	for rows.Next() {
+		var domain string
+		rows.Scan(&domain)
+		certnode.Domains = append(certnode.Domains, domain)
+	}
+
+	return certnode, nil
+}
+
+func (d *crtsh) CTexample(domain string) error {
+	s, err := d.QueryDomain(domain, false, false)
+	if err != nil {
+		return err
+	}
+
+	for i := range s {
+		fmt.Println(s[i].HexString(), " ", s[i].B64Encode())
+		cert, err := d.QueryCert(s[i])
+		if err != nil {
+			return err
+		}
+		for j := range cert.Domains {
+			fmt.Println("\t", cert.Domains[j])
+		}
+	}
+
+	return nil
+}

ct/Makefile

@@ -0,0 +1,11 @@
+package driver
+
+import (
+	"github.com/lanrat/certgraph/graph"
+)
+
+type Driver interface {
+	QueryDomain(domain string, include_expired bool, include_subdomains bool) ([]graph.Fingerprint, error)
+	QueryCert(fp graph.Fingerprint) (*graph.CertNode, error)
+	CTexample(domain string) error
+}

ct/main.go

@@ -3,7 +3,7 @@ package google
 /*
  * This file implements an unofficial API client for Google's
  * Certificate Transparency search
- * https://www.google.com/transparencyreport/https/ct/
+ * https://transparencyreport.google.com/https/certificates
  *
  * As the API is unofficial and has been reverse engineered it may stop working
  * at any time and comes with no guarantees.
@@ -19,21 +19,31 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/lanrat/certgraph/driver"
 	"github.com/lanrat/certgraph/graph"
 )
 
 // BASE URLs for Googl'e CT API
 const searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com"
 const searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF"
 const certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF"
-const MAX_PAGES = 50 // TODO: something better than hard-coding
 
-// global vars
-var jsonClient = &http.Client{Timeout: 10 * time.Second}
+type googleCT struct {
+	max_pages  float64
+	jsonClient *http.Client
+}
+
+func NewGoogleCTDriver() (driver.Driver, error) {
+	d := new(googleCT)
+	d.max_pages = 50 // TODO: make adjustable
+	d.jsonClient = &http.Client{Timeout: 10 * time.Second}
+
+	return d, nil
+}
 
 // gets JSON from url and parses it into target object
-func getJsonP(url string, target interface{}) error {
-	r, err := jsonClient.Get(url)
+func (d *googleCT) getJsonP(url string, target interface{}) error {
+	r, err := d.jsonClient.Get(url)
 	if err != nil {
 		return err
 	}
@@ -52,7 +62,7 @@ func getJsonP(url string, target interface{}) error {
 	return json.Unmarshal(respData, target)
 }
 
-func QueryDomain(domain string, include_expired bool, include_subdomains bool) ([]graph.Fingerprint, error) {
+func (d *googleCT) QueryDomain(domain string, include_expired bool, include_subdomains bool) ([]graph.Fingerprint, error) {
 	results := make([]graph.Fingerprint, 0, 5)
 
 	u, err := url.Parse(searchURL1)
@@ -74,8 +84,8 @@ func QueryDomain(domain string, include_expired bool, include_subdomains bool) (
 	// TODO allow for selective pagnation
 
 	// iterate over results
-	for len(nextURL) > 1 && currentPage <= MAX_PAGES {
-		err = getJsonP(nextURL, &raw)
+	for len(nextURL) > 1 && currentPage <= d.max_pages {
+		err = d.getJsonP(nextURL, &raw)
 		if err != nil {
 			return results, err
 		}
@@ -127,7 +137,7 @@ func QueryDomain(domain string, include_expired bool, include_subdomains bool) (
 	return results, nil
 }
 
-func GetCert(fp graph.Fingerprint) (*graph.CertNode, error) {
+func (d *googleCT) QueryCert(fp graph.Fingerprint) (*graph.CertNode, error) {
 	certnode := new(graph.CertNode)
 	certnode.Fingerprint = fp
 	certnode.Domains = make([]string, 0, 5)
@@ -144,7 +154,7 @@ func GetCert(fp graph.Fingerprint) (*graph.CertNode, error) {
 
 	var raw [][]interface{}
 
-	err = getJsonP(u.String(), &raw)
+	err = d.getJsonP(u.String(), &raw)
 	if err != nil {
 		return certnode, err
 	}
@@ -170,15 +180,15 @@ func GetCert(fp graph.Fingerprint) (*graph.CertNode, error) {
 }
 
 // example function to use Google's CT API
-func CTexample(domain string) error {
-	s, err := QueryDomain(domain, false, false)
+func (d *googleCT) CTexample(domain string) error {
+	s, err := d.QueryDomain(domain, false, false)
 	if err != nil {
 		return err
 	}
 
 	for i := range s {
 		fmt.Println(s[i].HexString(), " ", s[i].B64Encode())
-		cert, err := GetCert(s[i])
+		cert, err := d.QueryCert(s[i])
 		if err != nil {
 			return err
 		}