diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go index 97c16e712712..86b990948990 100644 --- a/credentials/xds/xds.go +++ b/credentials/xds/xds.go @@ -23,9 +23,7 @@ package xds import ( "context" "crypto/tls" - "crypto/x509" "errors" - "fmt" "net" "sync/atomic" "time" @@ -138,40 +136,6 @@ func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawCo if err != nil { return nil, nil, err } - cfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { - // Parse all raw certificates presented by the peer. - var certs []*x509.Certificate - for _, rc := range rawCerts { - cert, err := x509.ParseCertificate(rc) - if err != nil { - return err - } - certs = append(certs, cert) - } - - // Build the intermediates list and verify that the leaf certificate - // is signed by one of the root certificates. - intermediates := x509.NewCertPool() - for _, cert := range certs[1:] { - intermediates.AddCert(cert) - } - opts := x509.VerifyOptions{ - Roots: cfg.RootCAs, - Intermediates: intermediates, - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - } - if _, err := certs[0].Verify(opts); err != nil { - return err - } - // The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to - // only look at the SANs on the leaf cert. - if cert := certs[0]; !hi.MatchingSANExists(cert) { - // TODO: Print the complete certificate once the x509 package - // supports a String() method on the Certificate type. - return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs) - } - return nil - } // Perform the TLS handshake with the tls.Config that we have. We run the // actual Handshake() function in a goroutine because we need to respect the diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go index dcff7ad62223..81074bedb40e 100644 --- a/internal/credentials/xds/handshake_info.go +++ b/internal/credentials/xds/handshake_info.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc/attributes" "google.golang.org/grpc/credentials/tls/certprovider" "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/credentials/spiffe" "google.golang.org/grpc/internal/xds/matcher" "google.golang.org/grpc/resolver" ) @@ -144,6 +145,7 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config, return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) } cfg.RootCAs = km.Roots + cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, true) if idProv != nil { km, err := idProv.KeyMaterial(ctx) @@ -155,6 +157,60 @@ func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config, return cfg, nil } +func (hi *HandshakeInfo) buildVerifyFunc(km *certprovider.KeyMaterial, isClient bool) func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + // Parse all raw certificates presented by the peer. + var certs []*x509.Certificate + for _, rc := range rawCerts { + cert, err := x509.ParseCertificate(rc) + if err != nil { + return err + } + certs = append(certs, cert) + } + + // Build the intermediates list and verify that the leaf certificate is + // signed by one of the root certificates. If a SPIFFE Bundle Map is + // configured, it is used to get the root certs. Otherwise, the + // configured roots in the root provider are used. + intermediates := x509.NewCertPool() + for _, cert := range certs[1:] { + intermediates.AddCert(cert) + } + roots := km.Roots + // If a SPIFFE Bundle Map is configured, find the roots for the trust + // domain of the leaf certificate. + if km.SPIFFEBundleMap != nil { + var err error + roots, err = spiffe.GetRootsFromSPIFFEBundleMap(km.SPIFFEBundleMap, certs[0]) + if err != nil { + return err + } + } + opts := x509.VerifyOptions{ + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if isClient { + opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + } else { + opts.KeyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + } + if _, err := certs[0].Verify(opts); err != nil { + return err + } + // The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to + // only look at the SANs on the leaf cert. + if cert := certs[0]; !hi.MatchingSANExists(cert) { + // TODO: Print the complete certificate once the x509 package + // supports a String() method on the Certificate type. + return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs) + } + return nil + } +} + // ServerSideTLSConfig constructs a tls.Config to be used in a server-side // handshake based on the contents of the HandshakeInfo. func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) { @@ -186,7 +242,15 @@ func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, if err != nil { return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) } - cfg.ClientCAs = km.Roots + if km.SPIFFEBundleMap != nil && hi.requireClientCert { + // ClientAuth, if set greater than tls.RequireAnyClientCert, must be + // dropped to tls.RequireAnyClientCert so that custom verification + // to use SPIFFE Bundles is done. + cfg.ClientAuth = tls.RequireAnyClientCert + cfg.VerifyPeerCertificate = hi.buildVerifyFunc(km, false) + } else { + cfg.ClientCAs = km.Roots + } } return cfg, nil } diff --git a/internal/credentials/xds/handshake_info_test.go b/internal/credentials/xds/handshake_info_test.go index a3e598be3db3..53b2700b41ef 100644 --- a/internal/credentials/xds/handshake_info_test.go +++ b/internal/credentials/xds/handshake_info_test.go @@ -19,14 +19,23 @@ package xds import ( + "context" + "crypto/tls" "crypto/x509" + "fmt" "net" "net/netip" "net/url" + "os" "regexp" + "strings" "testing" + "time" + + "google.golang.org/grpc/testdata" "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/credentials/spiffe" "google.golang.org/grpc/internal/xds/matcher" ) @@ -34,6 +43,10 @@ type testCertProvider struct { certprovider.Provider } +type testCertProviderWithKeyMaterial struct { + certprovider.Provider +} + func TestDNSMatch(t *testing.T) { tests := []struct { desc string @@ -397,3 +410,86 @@ func TestEqual(t *testing.T) { }) } } + +func (p *testCertProviderWithKeyMaterial) KeyMaterial(_ context.Context) (*certprovider.KeyMaterial, error) { + km := &certprovider.KeyMaterial{} + spiffeBundleMapContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json")) + if err != nil { + return nil, err + } + bundleMap, err := spiffe.BundleMapFromBytes(spiffeBundleMapContents) + if err != nil { + return nil, err + } + km.SPIFFEBundleMap = bundleMap + rootFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/ca.pem")) + if err != nil { + return nil, err + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(rootFileContents) { + return nil, fmt.Errorf("Failed to parse root certificate") + } + km.Roots = trustPool + + certFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffe.pem")) + if err != nil { + return nil, err + } + keyFileContents, err := os.ReadFile(testdata.Path("spiffe_end2end/client.key")) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(certFileContents, keyFileContents) + if err != nil { + return nil, err + } + km.Certs = []tls.Certificate{cert} + return km, nil +} + +func TestBuildVerifyFuncFailures(t *testing.T) { + tests := []struct { + desc string + peerCertChain [][]byte + wantErr string + }{ + { + desc: "invalid x509", + peerCertChain: [][]byte{[]byte("NOT_A_CERT")}, + wantErr: "x509: malformed certificate", + }, + { + desc: "invalid SPIFFE ID in peer cert", + // server1.pem doesn't have a valid SPIFFE ID, so attempted to get a + // root from the SPIFFE Bundle Map will fail + peerCertChain: loadCert(t, testdata.Path("server1.pem"), testdata.Path("server1.key")), + wantErr: "spiffe: could not get spiffe ID from peer leaf cert but verification with spiffe trust map was configure", + }, + } + testProvider := testCertProviderWithKeyMaterial{} + hi := NewHandshakeInfo(&testProvider, &testProvider, nil, true) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + cfg, err := hi.ClientSideTLSConfig(ctx) + if err != nil { + t.Fatalf("hi.ClientSideTLSConfig() failed with err %v", err) + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + err = cfg.VerifyPeerCertificate(tc.peerCertChain, nil) + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("VerifyPeerCertificate got err %v, want: %v", err, tc.wantErr) + } + }) + } +} + +func loadCert(t *testing.T, certPath, keyPath string) [][]byte { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + t.Fatalf("LoadX509KeyPair(%s, %s) failed: %v", certPath, keyPath, err) + } + return cert.Certificate + +} diff --git a/internal/testutils/xds/e2e/bootstrap.go b/internal/testutils/xds/e2e/bootstrap.go index 0f39e3c7ad8d..d902e94a5144 100644 --- a/internal/testutils/xds/e2e/bootstrap.go +++ b/internal/testutils/xds/e2e/bootstrap.go @@ -44,6 +44,65 @@ func DefaultFileWatcherConfig(certPath, keyPath, caPath string) json.RawMessage }`, certPath, keyPath, caPath)) } +// SPIFFEFileWatcherConfig is a helper function to create a default certificate +// provider plugin configuration. The test is expected to have setup the files +// appropriately before this configuration is used to instantiate providers. +func SPIFFEFileWatcherConfig(certPath, keyPath, caPath, spiffeBundleMapPath string) json.RawMessage { + return json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": { + "certificate_file": %q, + "private_key_file": %q, + "ca_certificate_file": %q, + "spiffe_trust_bundle_map_file": %q, + "refresh_interval": "600s" + } + }`, certPath, keyPath, caPath, spiffeBundleMapPath)) +} + +// SPIFFEBootstrapContents creates a bootstrap configuration with the given node +// ID and server URI. It also creates certificate provider configuration using +// SPIFFE certificates and sets the listener resource name template to be used +// on the server side. +func SPIFFEBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { + t.Helper() + + // Create a directory to hold certs and key files used on the server side. + serverDir, err := createTmpDirWithCerts("testServerSideXDSSPIFFE*", "spiffe_end2end/server_spiffe.pem", "spiffe_end2end/server.key", "spiffe_end2end/ca.pem", "spiffe_end2end/server_spiffebundle.json") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a directory to hold certs and key files used on the client side. + clientDir, err := createTmpDirWithCerts("testClientSideXDSSPIFFE*", "spiffe_end2end/client_spiffe.pem", "spiffe_end2end/client.key", "spiffe_end2end/ca.pem", "spiffe_end2end/client_spiffebundle.json") + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create certificate providers section of the bootstrap config with entries + // for both the client and server sides. + cpc := map[string]json.RawMessage{ + ServerSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile), path.Join(serverDir, spiffeBundleMapFile)), + ClientSideCertProviderInstance: SPIFFEFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile), path.Join(clientDir, spiffeBundleMapFile)), + } + + // Create the bootstrap configuration. + bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{ + Servers: []byte(fmt.Sprintf(`[{ + "server_uri": "passthrough:///%s", + "channel_creds": [{"type": "insecure"}] + }]`, serverURI)), + Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)), + CertificateProviders: cpc, + ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + return bs + +} + // DefaultBootstrapContents creates a default bootstrap configuration with the // given node ID and server URI. It also creates certificate provider // configuration and sets the listener resource name template to be used on the @@ -52,13 +111,13 @@ func DefaultBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { t.Helper() // Create a directory to hold certs and key files used on the server side. - serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + serverDir, err := createTmpDirWithCerts("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem", "") if err != nil { t.Fatalf("Failed to create bootstrap configuration: %v", err) } // Create a directory to hold certs and key files used on the client side. - clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + clientDir, err := createTmpDirWithCerts("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem", "") if err != nil { t.Fatalf("Failed to create bootstrap configuration: %v", err) } @@ -88,9 +147,10 @@ func DefaultBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { const ( // Names of files inside tempdir, for certprovider plugin to watch. - certFile = "cert.pem" - keyFile = "key.pem" - rootFile = "ca.pem" + certFile = "cert.pem" + keyFile = "key.pem" + rootFile = "ca.pem" + spiffeBundleMapFile = "spiffe_bundle_map.json" ) func createTmpFile(src, dst string) error { @@ -109,7 +169,7 @@ func createTmpFile(src, dst string) error { // rootSrc files and creates appropriate files under the newly create tempDir. // Returns the path of the created tempDir if successful, and an error // otherwise. -func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc string) (string, error) { +func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc, spiffeBundleMapSrc string) (string, error) { // Create a temp directory. Passing an empty string for the first argument // uses the system temp directory. dir, err := os.MkdirTemp("", dirPattern) @@ -126,5 +186,10 @@ func createTmpDirWithCerts(dirPattern, certSrc, keySrc, rootSrc string) (string, if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil { return "", err } + if spiffeBundleMapSrc != "" { + if err := createTmpFile(testdata.Path(spiffeBundleMapSrc), path.Join(dir, spiffeBundleMapFile)); err != nil { + return "", err + } + } return dir, nil } diff --git a/internal/testutils/xds/e2e/setup/setup.go b/internal/testutils/xds/e2e/setup/setup.go index baf5aaf47f53..f1ac21b04046 100644 --- a/internal/testutils/xds/e2e/setup/setup.go +++ b/internal/testutils/xds/e2e/setup/setup.go @@ -59,3 +59,27 @@ func ManagementServerAndResolver(t *testing.T) (*e2e.ManagementServer, string, [ return xdsServer, nodeID, bc, r } + +// ManagementServerAndResolverWithSPIFFE is exactly the same as +// ManagementServerAndResolver, except that it uses a bootstrap configuration +// containing certificate providers utilizing SPIFFE test certificates. +func ManagementServerAndResolverWithSPIFFE(t *testing.T) (*e2e.ManagementServer, + string, []byte, resolver.Builder) { + // Start an xDS management server. + xdsServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + + // Create bootstrap configuration pointing to the above management server. + nodeID := uuid.New().String() + bc := e2e.SPIFFEBootstrapContents(t, nodeID, xdsServer.Address) + + // Create an xDS resolver with the above bootstrap configuration. + if internal.NewXDSResolverWithConfigForTesting == nil { + t.Fatalf("internal.NewXDSResolverWithConfigForTesting is nil") + } + r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + + return xdsServer, nodeID, bc, r +} diff --git a/test/xds/xds_client_certificate_providers_test.go b/test/xds/xds_client_certificate_providers_test.go index 7f5ade582663..03bcd603c812 100644 --- a/test/xds/xds_client_certificate_providers_test.go +++ b/test/xds/xds_client_certificate_providers_test.go @@ -350,3 +350,162 @@ func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) } } + +// Tests the case where the bootstrap configuration contains one certificate +// provider configured with SPIFFE Bundle Map roots on the client side, and xDS +// credentials with an insecure fallback is specified at dial time. The +// management server responds with three clusters: +// 1. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap, and the server uses a SPIFFE cert. +// 2. contains valid security configuration pointing to the certificate provider +// instance specified in the bootstrap, and the server uses a SPIFFE cert chain. +// 3. contains invalid security configuration pointing to a non-existent +// certificate provider instance +// +// The test verifies that RPCs to the first two clusters succeed, while RPCs to +// the third cluster fails with an appropriate code and error message. +func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfigurationSPIFFE(t *testing.T) { + mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t) + + // Create test backends for all three clusters + // backend1 configured with a SPIFFE cert, represents cluster1 + // backend2 configured with a SPIFFE cert chain, represents cluster2 + // backend3 configured with insecure creds, represents cluster3 + serverCreds := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert) + server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds)) + defer server1.Stop() + serverCreds2 := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert) + server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds2)) + defer server2.Stop() + server3 := stubserver.StartTestService(t, nil) + defer server3.Stop() + + // Configure client side xDS resources on the management server. + const serviceName = "my-service-client-side-xds" + const routeConfigName = "route-" + serviceName + const clusterName1 = "cluster1-" + serviceName + const clusterName2 = "cluster2-" + serviceName + const clusterName3 = "cluster3-" + serviceName + const endpointsName1 = "endpoints1-" + serviceName + const endpointsName2 = "endpoints2-" + serviceName + const endpointsName3 = "endpoints3-" + serviceName + listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)} + // Route configuration: + // - "/grpc.testing.TestService/EmptyCall" --> cluster1 + // - "/grpc.testing.TestService/UnaryCall" --> cluster2 + // - "/grpc.testing.TestService/FullDuplexCall" --> cluster3 + routes := []*v3routepb.RouteConfiguration{{ + Name: routeConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3}, + }}, + }, + }, + }}, + }} + // Clusters: + // - cluster1 with cert provider name e2e.ClientSideCertProviderInstance and mTLS. + // - cluster2 with cert provider name e2e.ClientSideCertProviderInstance and mTLS. + // - cluster3 with non-existent cert provider name. + clusters := []*v3clusterpb.Cluster{ + e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS), + e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelMTLS), + func() *v3clusterpb.Cluster { + cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS) + cluster3.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "non-existent-certificate-provider-instance-name", + }, + }, + }), + }, + } + return cluster3 + }(), + } + // Endpoints for each of the above clusters with backends created earlier. + endpoints := []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)}), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make an RPC to be routed to cluster1 and verify that it succeeds. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server1.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 1) + + // Make an RPC to be routed to cluster2 and verify that it succeeds. + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil { + t.Fatalf("UnaryCall() failed: %v", err) + } + if got, want := peer.Addr.String(), server2.Address; got != want { + t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want) + } + // In this call the server contains a peer chain of length 2 + verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 2) + + // Make an RPC to be routed to cluster3 and verify that it fails. + const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration` + if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr) + } +} diff --git a/test/xds/xds_server_integration_test.go b/test/xds/xds_server_integration_test.go index a63a75242f4d..3f3849db673e 100644 --- a/test/xds/xds_server_integration_test.go +++ b/test/xds/xds_server_integration_test.go @@ -30,6 +30,7 @@ import ( "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" xdscreds "google.golang.org/grpc/credentials/xds" "google.golang.org/grpc/internal" @@ -38,6 +39,7 @@ import ( "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/internal/testutils/xds/e2e/setup" + "google.golang.org/grpc/peer" "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" "google.golang.org/grpc/xds" @@ -421,3 +423,126 @@ func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) { t.Fatalf("rpc EmptyCall() failed: %v", err) } } + +// TestServerSideXDS_FileWatcherCertsSPIFFE is an e2e test which verifies xDS +// credentials with file watcher certificate provider that is configured with a +// SPIFFE Bundle Map for it's roots. +// +// The following sequence of events happen as part of this test: +// - An xDS-enabled gRPC server is created and xDS credentials are configured. +// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS +// credentials are configured. +// - Control plane is configured to send security configuration to both the +// client and the server, pointing to the file watcher certificate provider. +// We verify both TLS and mTLS scenarios. +func (s) TestServerSideXDS_FileWatcherCertsSPIFFE(t *testing.T) { + tests := []struct { + name string + secLevel e2e.SecurityLevel + }{ + { + name: "tls", + secLevel: e2e.SecurityLevelTLS, + }, + { + name: "mtls", + secLevel: e2e.SecurityLevelMTLS, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t) + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS + // resources corresponding to it. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + + // Create xDS resources to be consumed on the client side. This + // includes the listener, route configuration, cluster (with + // security configuration) and endpoint resources. + serviceName := "my-service-file-watcher-certs-" + test.name + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: test.secLevel, + }) + + // Create an inbound xDS listener resource for the server side that + // contains security configuration pointing to the file watcher + // plugin. + inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName") + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make an RPC. + cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver)) + if err != nil { + t.Fatalf("failed to create a client for server: %v", err) + } + defer cc.Close() + + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeerSPIFFE(t, peer, test.secLevel, 1) + }) + } +} + +// Checks the AuthInfo available in the peer if it matches the expected security +// level of the connection. +func verifySecurityInformationFromPeerSPIFFE(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel, wantPeerChainLen int) { + // This is not a true helper in the Go sense, because it does not perform + // setup or cleanup tasks. Marking it a helper is to ensure that when the + // test fails, the line information of the caller is outputted instead of + // from here. + // + // And this function directly calls t.Fatalf() instead of returning an error + // and letting the caller decide what to do with it. This is also OK since + // all callers will simply end up calling t.Fatalf() with the returned + // error, and can't add any contextual information of value to the error + // message. + t.Helper() + + authType := pr.AuthInfo.AuthType() + switch wantSecLevel { + case e2e.SecurityLevelNone: + if authType != "insecure" { + t.Fatalf("AuthType() is %s, want insecure", authType) + } + case e2e.SecurityLevelMTLS: + if authType != "tls" { + t.Fatalf("AuthType() is %s, want tls", authType) + } + ai, ok := pr.AuthInfo.(credentials.TLSInfo) + if !ok { + t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{}) + } + if len(ai.State.PeerCertificates) != wantPeerChainLen { + t.Fatalf("Number of peer certificates is %d, want %d", len(ai.State.PeerCertificates), wantPeerChainLen) + } + } +}