Performance impact of publicsuffix parsing #149
Description
The weppos publicsuffix package has a significant impact on the performance of zcrypto/x509 and zmap/zlint, https://github.com/globalsign/publicsuffix might help to improve performance significantly.
See the pprof snippet below where the publicsuffix.ParseFromListWithOptions call takes about 35% of the whole runtime.
See also:
- complete pprof diagram zcrypto-zmap-pprof.pdf
- issue Performance needs some work weppos/publicsuffix-go#133.
- x509_modified.go#L1484
(pprof) list ParseFromListWithOptions
Total: 1.78s
ROUTINE ======================== github.com/weppos/publicsuffix-go/publicsuffix.ParseFromListWithOptions in /home/digitorus/go/src/github.com/weppos/publicsuffix-go/publicsuffix/publicsuffix.go
0 620ms (flat, cum) 34.83% of Total
. . 453: n, err := normalize(name)
. . 454: if err != nil {
. . 455: return nil, err
. . 456: }
. . 457:
. 460ms 458: r := l.Find(n, options)
. 90ms 459: if tld := r.Decompose(n)[1]; tld == "" {
. . 460: return nil, fmt.Errorf("%s is a suffix", n)
. . 461: }
. . 462:
. . 463: dn := &DomainName{Rule: r}
. 70ms 464: dn.TLD, dn.SLD, dn.TRD = decompose(r, n)
. . 465: return dn, nil
. . 466:}
. . 467:
. . 468:func normalize(name string) (string, error) {
. . 469: ret := strings.ToLower(name)
Used test for profiling:
go test -bench=. -benchmem -cpuprofile profile.out
package main
import (
"encoding/pem"
"testing"
"github.com/zmap/zcrypto/x509"
"github.com/zmap/zlint"
)
const cert = `-----BEGIN CERTIFICATE-----
MIIF+TCCBOGgAwIBAgISAyn7mY0eJNhX/0WOSGr4+2i0MA0GCSqGSIb3DQEBCwUA
MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMzAeFw0xODA3MTcxNDIwMDlaFw0x
ODEwMTUxNDIwMDlaMBIxEDAOBgNVBAMTB3ptYXAuaW8wggEiMA0GCSqGSIb3DQEB
AQUAA4IBDwAwggEKAoIBAQDDyEPCKn+SDvBD/u5dLL0DIp+ApmcnpDActPJPq3iW
y21PZNDy4IYjfXw77r54D473Vic/l/SyVGw6XGKmjtaup2Z/70ZDDq0PCXLudQdZ
kWop/MC2KWFSUOtJC48XAxAR9sCK5TyWqme+epXKp6zId7zjE95gIiqyh8THNj1Y
0G819h0fAesiKrSJws8I0eUCseAReQMk6IUJNNU61tf+cni5XxEMYg8Ek0u/RjCE
H9b8OcXrb+3+wuaOtd4oQsgS+LHcDfFkioGogDlG1t49bDUFRI3FRNtPEwNker4z
ywtYGONJvJC0Pn5FtqZE8JhuCdxeRRk46ZobYnHJFODRAgMBAAGjggMPMIIDCzAO
BgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwG
A1UdEwEB/wQCMAAwHQYDVR0OBBYEFNFrSPZACAq+QmNXUllxRmJjPlPeMB8GA1Ud
IwQYMBaAFKhKamMEfd265tE5t6ZFZe/zqOyhMG8GCCsGAQUFBwEBBGMwYTAuBggr
BgEFBQcwAYYiaHR0cDovL29jc3AuaW50LXgzLmxldHNlbmNyeXB0Lm9yZzAvBggr
BgEFBQcwAoYjaHR0cDovL2NlcnQuaW50LXgzLmxldHNlbmNyeXB0Lm9yZy8wEgYD
VR0RBAswCYIHem1hcC5pbzCB/gYDVR0gBIH2MIHzMAgGBmeBDAECATCB5gYLKwYB
BAGC3xMBAQEwgdYwJgYIKwYBBQUHAgEWGmh0dHA6Ly9jcHMubGV0c2VuY3J5cHQu
b3JnMIGrBggrBgEFBQcCAjCBngyBm1RoaXMgQ2VydGlmaWNhdGUgbWF5IG9ubHkg
YmUgcmVsaWVkIHVwb24gYnkgUmVseWluZyBQYXJ0aWVzIGFuZCBvbmx5IGluIGFj
Y29yZGFuY2Ugd2l0aCB0aGUgQ2VydGlmaWNhdGUgUG9saWN5IGZvdW5kIGF0IGh0
dHBzOi8vbGV0c2VuY3J5cHQub3JnL3JlcG9zaXRvcnkvMIIBBAYKKwYBBAHWeQIE
AgSB9QSB8gDwAHYAKTxRllTIOWW6qlD8WAfUt2+/WHopctykwwz05UVH9HgAAAFk
qNPIygAABAMARzBFAiEAiBDl2llqZ+NAx+qKDR4Nz7skZFyXBCIIleZvqTEfOtYC
IAPJQoWeSH9dTrIamnoWMz0GcU5pmFiThRvV7lYzBjxFAHYAVYHUwhaQNgFK6gub
VzxT8MDkOHhwJQgXL6OqHQcT0wwAAAFkqNPKdQAABAMARzBFAiEA9cuUQMkCFTQc
T/c+xJSufZy4sCLBwVAkeaSqn/gRVX4CIFaMT0Y/HDP92CbcnioU42ycKC2Rp4av
IYylOSyBvA4sMA0GCSqGSIb3DQEBCwUAA4IBAQCDx1Tjl+IErHLZh3GqTINsbKg5
9UuwChKxoZIfaONl+Ot7pZwiMJeUrrE/1IJyPvItfpmCzKrucJEt5Y0iqWiC/kYt
GGA1qt1JyqYCXuQ5EAP7PRW4OrtfRXPZ8A4TEttU5NPw0bEbHde030soHiwvFSvn
NxN5jkMRHMVT6h5B9JcF9HAM0RRCfd8GKigbqrJkC6DJwQ+4VBU/e6JwQSti0S4T
YR/hQaPp410CUKxYrN68bI2HApw2qahzmIVCeo3gwBpickZsf+J+z9+VFlSHivVu
tuzpSShRCSSym392PUfxgkvfxy97+2pveJ45uAv7z/pfWIwYVUfgHktN92Bz
-----END CERTIFICATE-----`
func BenchmarkHandler(b *testing.B) {
block, _ := pem.Decode([]byte(cert))
for n := 0; n < b.N; n++ {
parsed, err := x509.ParseCertificate(block.Bytes)
if err != nil {
b.Fatal(err)
}
zlint.LintCertificate(parsed)
}
}