[Security] Add verification logic using SPIFFE Bundle Maps in XDS

credentials/xds/xds.go

@@ -23,9 +23,7 @@ package xds
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
 	"errors"
-	"fmt"
 	"net"
 	"sync/atomic"
 	"time"
@@ -138,40 +136,6 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo
 	if err != nil {
 		return nil, nil, err
 	}
-	cfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
-		// Parse all raw certificates presented by the peer.
-		var certs []*x509.Certificate
-		for _, rc := range rawCerts {
-			cert, err := x509.ParseCertificate(rc)
-			if err != nil {
-				return err
-			}
-			certs = append(certs, cert)
-		}
-
-		// Build the intermediates list and verify that the leaf certificate
-		// is signed by one of the root certificates.
-		intermediates := x509.NewCertPool()
-		for _, cert := range certs[1:] {
-			intermediates.AddCert(cert)
-		}
-		opts := x509.VerifyOptions{
-			Roots:         cfg.RootCAs,
-			Intermediates: intermediates,
-			KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		}
-		if _, err := certs[0].Verify(opts); err != nil {
-			return err
-		}
-		// The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to
-		// only look at the SANs on the leaf cert.
-		if cert := certs[0]; !hi.MatchingSANExists(cert) {
-			// TODO: Print the complete certificate once the x509 package
-			// supports a String() method on the Certificate type.
-			return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs)
-		}
-		return nil
-	}
 
 	// Perform the TLS handshake with the tls.Config that we have. We run the
 	// actual Handshake() function in a goroutine because we need to respect the

internal/credentials/xds/handshake_info.go

@@ -31,6 +31,7 @@ import (
 	"google.golang.org/grpc/attributes"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/credentials/spiffe"
 	"google.golang.org/grpc/internal/xds/matcher"
 	"google.golang.org/grpc/resolver"
 )
@@ -144,6 +145,7 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config,
 		return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
 	}
 	cfg.RootCAs = km.Roots
+	cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, true)
 
 	if idProv != nil {
 		km, err := idProv.KeyMaterial(ctx)
@@ -155,6 +157,60 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config,
 	return cfg, nil
 }
 
+func (hi *HandshakeInfo) buildVerifyFunc(km *certprovider.KeyMaterial, isClient bool) func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+	return func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+		// Parse all raw certificates presented by the peer.
+		var certs []*x509.Certificate
+		for _, rc := range rawCerts {
+			cert, err := x509.ParseCertificate(rc)
+			if err != nil {
+				return err
+			}
+			certs = append(certs, cert)
+		}
+
+		// Build the intermediates list and verify that the leaf certificate is
+		// signed by one of the root certificates. If a SPIFFE Bundle Map is
+		// configured, it is used to get the root certs. Otherwise, the
+		// configured roots in the root provider are used.
+		intermediates := x509.NewCertPool()
+		for _, cert := range certs[1:] {
+			intermediates.AddCert(cert)
+		}
+		roots := km.Roots
+		var err error
+		// If a SPIFFE Bundle Map is configured, find the roots for the trust
+		// domain of the leaf certificate.
+		if km.SPIFFEBundleMap != nil {
+			roots, err = spiffe.GetRootsFromSPIFFEBundleMap(km.SPIFFEBundleMap, certs[0])
+			if err != nil {
+				return err
+			}
+		}
+		opts := x509.VerifyOptions{
+			Roots:         roots,
+			Intermediates: intermediates,
+			KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		}
+		if isClient {
+			opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
+		} else {
+			opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
+		}
+		if _, err := certs[0].Verify(opts); err != nil {
+			return err
+		}
+		// The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to
+		// only look at the SANs on the leaf cert.
+		if cert := certs[0]; !hi.MatchingSANExists(cert) {
+			// TODO: Print the complete certificate once the x509 package
+			// supports a String() method on the Certificate type.
+			return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs)
+		}
+		return nil
+	}
+}
+
 // ServerSideTLSConfig constructs a tls.Config to be used in a server-side
 // handshake based on the contents of the HandshakeInfo.
 func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) {
@@ -186,7 +242,15 @@ func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config,
 		if err != nil {
 			return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err)
 		}
-		cfg.ClientCAs = km.Roots
+		if km.SPIFFEBundleMap != nil && hi.requireClientCert {
+			// ClientAuth, if set greater than tls.RequireAnyClientCert, must be
+			// dropped to tls.RequireAnyClientCert so that custom verification
+			// to use SPIFFE Bundles is done.
+			cfg.ClientAuth = tls.RequireAnyClientCert
+			cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, false)
+		} else {
+			cfg.ClientCAs = km.Roots
+		}
 	}
 	return cfg, nil
 }

internal/credentials/xds/handshake_info_test.go

@@ -19,21 +19,34 @@
 package xds
 
 import (
+	"context"
+	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"net"
 	"net/netip"
 	"net/url"
+	"os"
 	"regexp"
+	"strings"
 	"testing"
+	"time"
+
+	"google.golang.org/grpc/testdata"
 
 	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/credentials/spiffe"
 	"google.golang.org/grpc/internal/xds/matcher"
 )
 
 type testCertProvider struct {
 	certprovider.Provider
 }
 
+type testCertProviderWithKeyMaterial struct {
+	certprovider.Provider
+}
+
 func TestDNSMatch(t *testing.T) {
 	tests := []struct {
 		desc      string
@@ -397,3 +410,87 @@ func TestEqual(t *testing.T) {
 		})
 	}
 }
+
+func (p *testCertProviderWithKeyMaterial) KeyMaterial(_ context.Context) (*certprovider.KeyMaterial, error) {
+	km := &certprovider.KeyMaterial{}
+	spiffeBundleMapContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json"))
+	if err != nil {
+		return nil, err
+	}
+	bundleMap, err := spiffe.BundleMapFromBytes(spiffeBundleMapContents)
+	if err != nil {
+		return nil, err
+	}
+	km.SPIFFEBundleMap = bundleMap
+	rootFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem"))
+	if err != nil {
+		return nil, err
+	}
+	trustPool := x509.NewCertPool()
+	if !trustPool.AppendCertsFromPEM(rootFileContents) {
+		return nil, fmt.Errorf("Failed to parse root certificate")
+	}
+	km.Roots = trustPool
+
+	certFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffe.pem"))
+	if err != nil {
+		return nil, err
+	}
+	keyFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client.key"))
+	if err != nil {
+		return nil, err
+	}
+	cert, err := tls.X509KeyPair(certFileContents, keyFileContents)
+	if err != nil {
+		return nil, err
+	}
+	km.Certs = []tls.Certificate{cert}
+	return km, nil
+}
+
+func TestBuildVerifyFuncFailures(t *testing.T) {
+	tests := []struct {
+		desc          string
+		peerCertChain [][]byte
+		wantErr       string
+	}{
+		{
+			desc:          "invalid x509",
+			peerCertChain: [][]byte{[]byte("NOT_A_CERT")},
+			wantErr:       "x509: malformed certificate",
+		},
+		{
+			desc: "invalid x509",
+			// server1.pem doesn't have a valid SPIFFE ID, so attempted to get a
+			// root from the SPIFFE Bundle Map will fail
+			peerCertChain: loadCert(t, testdata.Path("server1.pem"),
+				testdata.Path("server1.key")),
+			wantErr: "spiffe: could not get spiffe ID from peer leaf cert but verification with spiffe trust map was configure",
+		},
+	}
+	testProvider := testCertProviderWithKeyMaterial{}
+	hi := NewHandshakeInfo(&testProvider, &testProvider, nil, true)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	cfg, err := hi.ClientSideTLSConfig(ctx)
+	if err != nil {
+		t.Fatalf("hi.ClientSideTLSConfig() failed with err %v", err)
+	}
+	for _, tc := range tests {
+		t.Run(tc.desc, func(t *testing.T) {
+			err = cfg.VerifyPeerCertificate(tc.peerCertChain, nil)
+			if !strings.Contains(err.Error(), tc.wantErr) {
+				t.Errorf("VerifyPeerCertificate got err %v, want: %v", err, tc.wantErr)
+			}
+		})
+	}
+}
+
+func loadCert(t *testing.T, certPath, keyPath string) [][]byte {
+	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		t.Fatalf("LoadX509KeyPair(%s, %s) failed: %v", certPath, keyPath, err)
+	}
+	return cert.Certificate
+
+}

internal/testutils/xds/e2e/bootstrap.go

@@ -44,6 +44,68 @@
 		}`, certPath, keyPath, caPath))
 }
 
+// SPIFFEFileWatcherConfig is a helper function to create a default certificate
+// provider plugin configuration. The test is expected to have setup the files
+// appropriately before this configuration is used to instantiate providers.
+func SPIFFEFileWatcherConfig(certPath, keyPath, caPath, spiffeBundleMapPath string) json.RawMessage {
+	return json.RawMessage(fmt.Sprintf(`{
+			"plugin_name": "file_watcher",
+			"config": {
+				"certificate_file": %q,
+				"private_key_file": %q,
+				"ca_certificate_file": %q,
+				"spiffe_trust_bundle_map_file": %q,
+				"refresh_interval": "600s"
+			}
+		}`, certPath, keyPath, caPath, spiffeBundleMapPath))
+}
+
+// SPIFFEBootstrapContents creates a bootstrap configuration with the given node
+// ID and server URI. It also creates certificate provider configuration using
+// SPIFFE certificates and sets the listener resource name template to be used
+// on the server side.
+func SPIFFEBootstrapContents(t *testing.T, nodeID, serverURI string) []byte {
+	t.Helper()
+
+	// Create a directory to hold certs and key files used on the server side.
+	var serverDir string
+	var err error
+
+	serverDir, err = createTmpDirWithCerts("testServerSideXDSSPIFFE*", "spiffe_end2end/server_spiffe.pem", "spiffe_end2end/server.key", "spiffe_end2end/ca.pem", "spiffe_end2end/server_spiffebundle.json")
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a directory to hold certs and key files used on the client side.
+	clientDir, err := createTmpDirWithCerts("testClientSideXDSSPIFFE*", "spiffe_end2end/client_spiffe.pem", "spiffe_end2end/client.key", "spiffe_end2end/ca.pem", "spiffe_end2end/client_spiffebundle.json")
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create certificate providers section of the bootstrap config with entries
+	// for both the client and server sides.
+	cpc := map[string]json.RawMessage{
+		ServerSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile), path.Join(serverDir, spiffeBundleMapFile)),
+		ClientSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile), path.Join(clientDir, spiffeBundleMapFile)),
+	}
+
+	// Create the bootstrap configuration.
+	bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": "passthrough:///%s",
+			"channel_creds": [{"type": "insecure"}]
+		}]`, serverURI)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		CertificateProviders:               cpc,
+		ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+	return bs
+
+}
+
 // DefaultBootstrapContents creates a default bootstrap configuration with the
 // given node ID and server URI. It also creates certificate provider
 // configuration and sets the listener resource name template to be used on the
@@ -52,13 +114,13 @@
 	t.Helper()
 
 	// Create a directory to hold certs and key files used on the server side.
-	serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem", "")
 	if err != nil {
 		t.Fatalf("Failed to create bootstrap configuration: %v", err)
 	}
 
 	// Create a directory to hold certs and key files used on the client side.
-	clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
+	clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem", "")
 	if err != nil {
 		t.Fatalf("Failed to create bootstrap configuration: %v", err)
 	}
@@ -88,9 +150,10 @@
 
 const (
 	// Names of files inside tempdir, for certprovider plugin to watch.
-	certFile = "cert.pem"
-	keyFile  = "key.pem"
-	rootFile = "ca.pem"
+	certFile            = "cert.pem"
+	keyFile             = "key.pem"
+	rootFile            = "ca.pem"
+	spiffeBundleMapFile = "spiffe_bundle_map.json"
 )
 
 func createTmpFile(src, dst string) error {
@@ -109,7 +172,7 @@
 // rootSrc files and creates appropriate files under the newly create tempDir.
 // Returns the path of the created tempDir if successful, and an error
 // otherwise.
-func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc string) (string, error) {
+func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc, spiffeBundleMapSrc string) (string, error) {
 	// Create a temp directory. Passing an empty string for the first argument
 	// uses the system temp directory.
 	dir, err := os.MkdirTemp("", dirPattern)
@@ -126,5 +189,10 @@
 	if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil {
 		return "", err
 	}
+	if spiffeBundleMapSrc != "" {
+		if err := createTmpFile(testdata.Path(spiffeBundleMapSrc), path.Join(dir, spiffeBundleMapFile)); err != nil {
+			return "", err
+		}
+	}
 	return dir, nil
 }

internal/testutils/xds/e2e/clientresources.go

@@ -143,6 +143,7 @@ func marshalAny(m proto.Message) *anypb.Any {
 // DefaultServerListener returns a basic xds Listener resource to be used on the
 // server side. The returned Listener resource contains an inline route
 // configuration with the name of routeName.
+// TODO gregorycooke follow security level, it goes into filter chain
 func DefaultServerListener(host string, port uint32, secLevel SecurityLevel, routeName string) *v3listenerpb.Listener {
 	return defaultServerListenerCommon(host, port, secLevel, routeName, true)
 }