Skip to content

Commit 58d1a72

Browse files
gtcooke94gtcooke94
authored
[Security] Add verification logic using SPIFFE Bundle Maps in XDS (#8229)
Add verification logic using SPIFFE Bundle Maps in XDS
1 parent f7d488d commit 58d1a72

File tree

7 files changed

+540
-43
lines changed

7 files changed

+540
-43
lines changed

‎credentials/xds/xds.go‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,7 @@ package xds
2323
import (
2424
"context"
2525
"crypto/tls"
26-
"crypto/x509"
2726
"errors"
28-
"fmt"
2927
"net"
3028
"sync/atomic"
3129
"time"
@@ -138,40 +136,6 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
138136
if err != nil {
139137
return nil, nil, err
140138
}
141-
cfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
142-
// Parse all raw certificates presented by the peer.
143-
var certs []*x509.Certificate
144-
for _, rc := range rawCerts {
145-
cert, err := x509.ParseCertificate(rc)
146-
if err != nil {
147-
return err
148-
}
149-
certs = append(certs, cert)
150-
}
151-
152-
// Build the intermediates list and verify that the leaf certificate
153-
// is signed by one of the root certificates.
154-
intermediates := x509.NewCertPool()
155-
for _, cert := range certs[1:] {
156-
intermediates.AddCert(cert)
157-
}
158-
opts := x509.VerifyOptions{
159-
Roots: cfg.RootCAs,
160-
Intermediates: intermediates,
161-
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
162-
}
163-
if _, err := certs[0].Verify(opts); err != nil {
164-
return err
165-
}
166-
// The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to
167-
// only look at the SANs on the leaf cert.
168-
if cert := certs[0]; !hi.MatchingSANExists(cert) {
169-
// TODO: Print the complete certificate once the x509 package
170-
// supports a String() method on the Certificate type.
171-
return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs)
172-
}
173-
return nil
174-
}
175139

176140
// Perform the TLS handshake with the tls.Config that we have. We run the
177141
// actual Handshake() function in a goroutine because we need to respect the

‎internal/credentials/xds/handshake_info.go‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"google.golang.org/grpc/attributes"
3232
"google.golang.org/grpc/credentials/tls/certprovider"
3333
"google.golang.org/grpc/internal"
34+
"google.golang.org/grpc/internal/credentials/spiffe"
3435
"google.golang.org/grpc/internal/xds/matcher"
3536
"google.golang.org/grpc/resolver"
3637
)
@@ -144,6 +145,7 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config,
144145
return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
145146
}
146147
cfg.RootCAs = km.Roots
148+
cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, true)
147149

148150
if idProv != nil {
149151
km, err := idProv.KeyMaterial(ctx)
@@ -155,6 +157,60 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config,
155157
return cfg, nil
156158
}
157159

160+
func (hi *HandshakeInfo) buildVerifyFunc(km *certprovider.KeyMaterial, isClient bool) func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
161+
return func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
162+
// Parse all raw certificates presented by the peer.
163+
var certs []*x509.Certificate
164+
for _, rc := range rawCerts {
165+
cert, err := x509.ParseCertificate(rc)
166+
if err != nil {
167+
return err
168+
}
169+
certs = append(certs, cert)
170+
}
171+
172+
// Build the intermediates list and verify that the leaf certificate is
173+
// signed by one of the root certificates. If a SPIFFE Bundle Map is
174+
// configured, it is used to get the root certs. Otherwise, the
175+
// configured roots in the root provider are used.
176+
intermediates := x509.NewCertPool()
177+
for _, cert := range certs[1:] {
178+
intermediates.AddCert(cert)
179+
}
180+
roots := km.Roots
181+
// If a SPIFFE Bundle Map is configured, find the roots for the trust
182+
// domain of the leaf certificate.
183+
if km.SPIFFEBundleMap != nil {
184+
var err error
185+
roots, err = spiffe.GetRootsFromSPIFFEBundleMap(km.SPIFFEBundleMap, certs[0])
186+
if err != nil {
187+
return err
188+
}
189+
}
190+
opts := x509.VerifyOptions{
191+
Roots: roots,
192+
Intermediates: intermediates,
193+
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
194+
}
195+
if isClient {
196+
opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
197+
} else {
198+
opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
199+
}
200+
if _, err := certs[0].Verify(opts); err != nil {
201+
return err
202+
}
203+
// The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to
204+
// only look at the SANs on the leaf cert.
205+
if cert := certs[0]; !hi.MatchingSANExists(cert) {
206+
// TODO: Print the complete certificate once the x509 package
207+
// supports a String() method on the Certificate type.
208+
return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs)
209+
}
210+
return nil
211+
}
212+
}
213+
158214
// ServerSideTLSConfig constructs a tls.Config to be used in a server-side
159215
// handshake based on the contents of the HandshakeInfo.
160216
func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) {
@@ -186,7 +242,15 @@ func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config,
186242
if err != nil {
187243
return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
188244
}
189-
cfg.ClientCAs = km.Roots
245+
if km.SPIFFEBundleMap != nil && hi.requireClientCert {
246+
// ClientAuth, if set greater than tls.RequireAnyClientCert, must be
247+
// dropped to tls.RequireAnyClientCert so that custom verification
248+
// to use SPIFFE Bundles is done.
249+
cfg.ClientAuth = tls.RequireAnyClientCert
250+
cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, false)
251+
} else {
252+
cfg.ClientCAs = km.Roots
253+
}
190254
}
191255
return cfg, nil
192256
}

‎internal/credentials/xds/handshake_info_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,34 @@
1919
package xds
2020

2121
import (
22+
"context"
23+
"crypto/tls"
2224
"crypto/x509"
25+
"fmt"
2326
"net"
2427
"net/netip"
2528
"net/url"
29+
"os"
2630
"regexp"
31+
"strings"
2732
"testing"
33+
"time"
34+
35+
"google.golang.org/grpc/testdata"
2836

2937
"google.golang.org/grpc/credentials/tls/certprovider"
38+
"google.golang.org/grpc/internal/credentials/spiffe"
3039
"google.golang.org/grpc/internal/xds/matcher"
3140
)
3241

3342
type testCertProvider struct {
3443
certprovider.Provider
3544
}
3645

46+
type testCertProviderWithKeyMaterial struct {
47+
certprovider.Provider
48+
}
49+
3750
func TestDNSMatch(t *testing.T) {
3851
tests := []struct {
3952
desc string
@@ -397,3 +410,86 @@ func TestEqual(t *testing.T) {
397410
})
398411
}
399412
}
413+
414+
func (p *testCertProviderWithKeyMaterial) KeyMaterial(_ context.Context) (*certprovider.KeyMaterial, error) {
415+
km := &certprovider.KeyMaterial{}
416+
spiffeBundleMapContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json"))
417+
if err != nil {
418+
return nil, err
419+
}
420+
bundleMap, err := spiffe.BundleMapFromBytes(spiffeBundleMapContents)
421+
if err != nil {
422+
return nil, err
423+
}
424+
km.SPIFFEBundleMap = bundleMap
425+
rootFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem"))
426+
if err != nil {
427+
return nil, err
428+
}
429+
trustPool := x509.NewCertPool()
430+
if !trustPool.AppendCertsFromPEM(rootFileContents) {
431+
return nil, fmt.Errorf("Failed to parse root certificate")
432+
}
433+
km.Roots = trustPool
434+
435+
certFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffe.pem"))
436+
if err != nil {
437+
return nil, err
438+
}
439+
keyFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client.key"))
440+
if err != nil {
441+
return nil, err
442+
}
443+
cert, err := tls.X509KeyPair(certFileContents, keyFileContents)
444+
if err != nil {
445+
return nil, err
446+
}
447+
km.Certs = []tls.Certificate{cert}
448+
return km, nil
449+
}
450+
451+
func TestBuildVerifyFuncFailures(t *testing.T) {
452+
tests := []struct {
453+
desc string
454+
peerCertChain [][]byte
455+
wantErr string
456+
}{
457+
{
458+
desc: "invalid x509",
459+
peerCertChain: [][]byte{[]byte("NOT_A_CERT")},
460+
wantErr: "x509: malformed certificate",
461+
},
462+
{
463+
desc: "invalid SPIFFE ID in peer cert",
464+
// server1.pem doesn't have a valid SPIFFE ID, so attempted to get a
465+
// root from the SPIFFE Bundle Map will fail
466+
peerCertChain: loadCert(t, testdata.Path("server1.pem"), testdata.Path("server1.key")),
467+
wantErr: "spiffe: could not get spiffe ID from peer leaf cert but verification with spiffe trust map was configure",
468+
},
469+
}
470+
testProvider := testCertProviderWithKeyMaterial{}
471+
hi := NewHandshakeInfo(&testProvider, &testProvider, nil, true)
472+
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
473+
defer cancel()
474+
cfg, err := hi.ClientSideTLSConfig(ctx)
475+
if err != nil {
476+
t.Fatalf("hi.ClientSideTLSConfig() failed with err %v", err)
477+
}
478+
for _, tc := range tests {
479+
t.Run(tc.desc, func(t *testing.T) {
480+
err = cfg.VerifyPeerCertificate(tc.peerCertChain, nil)
481+
if !strings.Contains(err.Error(), tc.wantErr) {
482+
t.Errorf("VerifyPeerCertificate got err %v, want: %v", err, tc.wantErr)
483+
}
484+
})
485+
}
486+
}
487+
488+
func loadCert(t *testing.T, certPath, keyPath string) [][]byte {
489+
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
490+
if err != nil {
491+
t.Fatalf("LoadX509KeyPair(%s, %s) failed: %v", certPath, keyPath, err)
492+
}
493+
return cert.Certificate
494+
495+
}

‎internal/testutils/xds/e2e/bootstrap.go‎

Lines changed: 71 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,65 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) json.RawMessage
4444
}`, certPath, keyPath, caPath))
4545
}
4646

47+
// SPIFFEFileWatcherConfig is a helper function to create a default certificate
48+
// provider plugin configuration. The test is expected to have setup the files
49+
// appropriately before this configuration is used to instantiate providers.
50+
func SPIFFEFileWatcherConfig(certPath, keyPath, caPath, spiffeBundleMapPath string) json.RawMessage {
51+
return json.RawMessage(fmt.Sprintf(`{
52+
"plugin_name": "file_watcher",
53+
"config": {
54+
"certificate_file": %q,
55+
"private_key_file": %q,
56+
"ca_certificate_file": %q,
57+
"spiffe_trust_bundle_map_file": %q,
58+
"refresh_interval": "600s"
59+
}
60+
}`, certPath, keyPath, caPath, spiffeBundleMapPath))
61+
}
62+
63+
// SPIFFEBootstrapContents creates a bootstrap configuration with the given node
64+
// ID and server URI. It also creates certificate provider configuration using
65+
// SPIFFE certificates and sets the listener resource name template to be used
66+
// on the server side.
67+
func SPIFFEBootstrapContents(t *testing.T, nodeID, serverURI string) []byte {
68+
t.Helper()
69+
70+
// Create a directory to hold certs and key files used on the server side.
71+
serverDir, err := createTmpDirWithCerts("testServerSideXDSSPIFFE*", "spiffe_end2end/server_spiffe.pem", "spiffe_end2end/server.key", "spiffe_end2end/ca.pem", "spiffe_end2end/server_spiffebundle.json")
72+
if err != nil {
73+
t.Fatalf("Failed to create bootstrap configuration: %v", err)
74+
}
75+
76+
// Create a directory to hold certs and key files used on the client side.
77+
clientDir, err := createTmpDirWithCerts("testClientSideXDSSPIFFE*", "spiffe_end2end/client_spiffe.pem", "spiffe_end2end/client.key", "spiffe_end2end/ca.pem", "spiffe_end2end/client_spiffebundle.json")
78+
if err != nil {
79+
t.Fatalf("Failed to create bootstrap configuration: %v", err)
80+
}
81+
82+
// Create certificate providers section of the bootstrap config with entries
83+
// for both the client and server sides.
84+
cpc := map[string]json.RawMessage{
85+
ServerSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile), path.Join(serverDir, spiffeBundleMapFile)),
86+
ClientSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile), path.Join(clientDir, spiffeBundleMapFile)),
87+
}
88+
89+
// Create the bootstrap configuration.
90+
bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
91+
Servers: []byte(fmt.Sprintf(`[{
92+
"server_uri": "passthrough:///%s",
93+
"channel_creds": [{"type": "insecure"}]
94+
}]`, serverURI)),
95+
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
96+
CertificateProviders: cpc,
97+
ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate,
98+
})
99+
if err != nil {
100+
t.Fatalf("Failed to create bootstrap configuration: %v", err)
101+
}
102+
return bs
103+
104+
}
105+
47106
// DefaultBootstrapContents creates a default bootstrap configuration with the
48107
// given node ID and server URI. It also creates certificate provider
49108
// configuration and sets the listener resource name template to be used on the
@@ -52,13 +111,13 @@ func DefaultBootstrapContents(t *testing.T, nodeID, serverURI string) []byte {
52111
t.Helper()
53112

54113
// Create a directory to hold certs and key files used on the server side.
55-
serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
114+
serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem", "")
56115
if err != nil {
57116
t.Fatalf("Failed to create bootstrap configuration: %v", err)
58117
}
59118

60119
// Create a directory to hold certs and key files used on the client side.
61-
clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
120+
clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem", "")
62121
if err != nil {
63122
t.Fatalf("Failed to create bootstrap configuration: %v", err)
64123
}
@@ -88,9 +147,10 @@ func DefaultBootstrapContents(t *testing.T, nodeID, serverURI string) []byte {
88147

89148
const (
90149
// Names of files inside tempdir, for certprovider plugin to watch.
91-
certFile = "cert.pem"
92-
keyFile = "key.pem"
93-
rootFile = "ca.pem"
150+
certFile = "cert.pem"
151+
keyFile = "key.pem"
152+
rootFile = "ca.pem"
153+
spiffeBundleMapFile = "spiffe_bundle_map.json"
94154
)
95155

96156
func createTmpFile(src, dst string) error {
@@ -109,7 +169,7 @@ func createTmpFile(src, dst string) error {
109169
// rootSrc files and creates appropriate files under the newly create tempDir.
110170
// Returns the path of the created tempDir if successful, and an error
111171
// otherwise.
112-
func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc string) (string, error) {
172+
func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc, spiffeBundleMapSrc string) (string, error) {
113173
// Create a temp directory. Passing an empty string for the first argument
114174
// uses the system temp directory.
115175
dir, err := os.MkdirTemp("", dirPattern)
@@ -126,5 +186,10 @@ func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc string) (string,
126186
if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil {
127187
return "", err
128188
}
189+
if spiffeBundleMapSrc != "" {
190+
if err := createTmpFile(testdata.Path(spiffeBundleMapSrc), path.Join(dir, spiffeBundleMapFile)); err != nil {
191+
return "", err
192+
}
193+
}
129194
return dir, nil
130195
}

0 commit comments

Comments
 (0)