starting to perform deduplication of discovered Organizations

Author: caffix

engine/plugins/rdap/plugin.go

@@ -243,7 +243,7 @@ func (rd *rdapPlugin) storeEntity(e *et.Event, level int, entity *rdap.Entity, a
 				_ = rd.createContactEdge(e.Session, cr, a, &general.SimpleRelation{Name: "person"}, src)
 			}
 		}
-	} else if name != "" && m.IsMatch(string(oam.Organization)) {
+	} else if m.IsMatch(string(oam.Organization)) && name != "" && !support.OrgNameExistsInContactRecord(e.Session, cr, name) {
 		if a, err := e.Session.Cache().CreateAsset(&org.Organization{
 			ID:   uuid.New().String(),
 			Name: name,

engine/plugins/support/database.go

@@ -20,6 +20,7 @@ import (
 	oamcert "github.com/owasp-amass/open-asset-model/certificate"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
 	"github.com/owasp-amass/open-asset-model/general"
+	"github.com/owasp-amass/open-asset-model/org"
 	"github.com/owasp-amass/open-asset-model/platform"
 	oamreg "github.com/owasp-amass/open-asset-model/registration"
 )
@@ -233,3 +234,20 @@ func CreateServiceAsset(session et.Session, src *dbt.Entity, rel oam.Relation, s
 	})
 	return result, err
 }
+
+func OrgNameExistsInContactRecord(session et.Session, cr *dbt.Entity, name string) bool {
+	if cr == nil {
+		return false
+	}
+
+	if edges, err := session.Cache().OutgoingEdges(cr, time.Time{}, "organization"); err == nil && len(edges) > 0 {
+		for _, edge := range edges {
+			if a, err := session.Cache().FindEntityById(edge.ToEntity.ID); err == nil && a != nil {
+				if org, ok := a.Asset.(*org.Organization); ok && strings.EqualFold(org.Name, name) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

engine/plugins/whois/domain_record.go

@@ -224,7 +224,8 @@ func (r *domrec) storeContact(e *et.Event, c *domrecContact, dr *dbt.Entity, m *
 			}
 		}
 	}
-	if wc.Organization != "" && m.IsMatch(string(oam.Organization)) {
+	if m.IsMatch(string(oam.Organization)) && wc.Organization != "" &&
+		!support.OrgNameExistsInContactRecord(e.Session, cr, wc.Organization) {
 		if a, err := e.Session.Cache().CreateAsset(&org.Organization{
 			ID:   uuid.New().String(),
 			Name: wc.Organization,

utils/viz/viz.go

@@ -215,9 +215,9 @@ func newNode(db repository.Repository, idx int, a *types.Entity, since time.Time
 		parts := []string{v.BuildingNumber, v.StreetName, v.City, v.Province, v.PostalCode}
 		key = strings.Join(parts, " ")
 	case *org.Organization:
-		key = fmt.Sprintf("%s, %s", v.Name, v.ID)
+		key = fmt.Sprintf("%s (%s)", v.Name, v.ID)
 	case *oamcert.TLSCertificate:
-		key = fmt.Sprintf("%s, %s", v.SubjectCommonName, v.SerialNumber)
+		key = fmt.Sprintf("%s (%s)", v.SubjectCommonName, v.SerialNumber)
 	}
 	title := fmt.Sprintf("%s: %s", atype, key)