xds: don't fail channel/server startup when xds creds is specified, but bootstrap is missing certificate providers (#6848)

Author: easwars

internal/testutils/xds/e2e/setup_certs.go

@@ -87,11 +87,35 @@ func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
 	}
 	roots := x509.NewCertPool()
 	if !roots.AppendCertsFromPEM(b) {
-		t.Fatal("failed to append certificates")
+		t.Fatal("Failed to append certificates")
 	}
 	return credentials.NewTLS(&tls.Config{
 		Certificates: []tls.Certificate{cert},
 		RootCAs:      roots,
 		ServerName:   "x.test.example.com",
 	})
 }
+
+// CreateServerTLSCredentials creates server-side TLS transport credentials
+// using certificate and key files from testdata/x509 directory.
+func CreateServerTLSCredentials(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatalf("tls.LoadX509KeyPair(x509/server1_cert.pem, x509/server1_key.pem) failed: %v", err)
+	}
+	b, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("os.ReadFile(x509/client_ca_cert.pem) failed: %v", err)
+	}
+	ca := x509.NewCertPool()
+	if !ca.AppendCertsFromPEM(b) {
+		t.Fatal("Failed to append certificates")
+	}
+	return credentials.NewTLS(&tls.Config{
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    ca,
+	})
+}

test/xds/xds_client_certificate_providers_test.go

@@ -0,0 +1,362 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/bootstrap"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Tests the case where the bootstrap configuration contains no certificate
+// providers, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server is configured to return client side xDS resources
+// with no security configuration. The test verifies that the gRPC client is
+// able to make RPCs to the backend which is configured to accept plaintext
+// connections. This ensures that the insecure fallback credentials are getting
+// used on the client.
+func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
+	if err != nil {
+		t.Fatalf("Failed to start management server: %v", err)
+	}
+	defer mgmtServer.Stop()
+
+	// Create bootstrap configuration with no certificate providers.
+	nodeID := uuid.New().String()
+	bs, err := bootstrap.Contents(bootstrap.Options{
+		NodeID:    nodeID,
+		ServerURI: mgmtServer.Address,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	newResolver := internal.NewXDSResolverWithConfigForTesting
+	if newResolver == nil {
+		t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset")
+	}
+	resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server, with no
+	// security configuration in the Cluster resource.
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains no certificate
+// providers, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server is configured to return client side xDS resources
+// with an mTLS security configuration. The test verifies that the gRPC client
+// moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and
+// string. This ensures that when the certificate provider instance name
+// specified in the security configuration is not present in the bootstrap,
+// channel creation does not fail, but it moves to TRANSIENT_FAILURE and
+// subsequent rpcs fail.
+func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
+	if err != nil {
+		t.Fatalf("Failed to start management server: %v", err)
+	}
+	defer mgmtServer.Stop()
+
+	// Create bootstrap configuration with no certificate providers.
+	nodeID := uuid.New().String()
+	bs, err := bootstrap.Contents(bootstrap.Options{
+		NodeID:    nodeID,
+		ServerURI: mgmtServer.Address,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	newResolver := internal.NewXDSResolverWithConfigForTesting
+	if newResolver == nil {
+		t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset")
+	}
+	resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server, with mTLS
+	// security configuration in the Cluster resource.
+	const serviceName = "my-service-client-side-xds"
+	const clusterName = "cluster-" + serviceName
+	const endpointsName = "endpoints-" + serviceName
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE.
+	cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Make an RPC and ensure that expected error is returned.
+	wantErr := fmt.Sprintf("identitiy certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance)
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains one certificate
+// provider, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server responds with three clusters:
+//  1. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap
+//  2. contains no security configuration, hence should use insecure fallback
+//  3. contains invalid security configuration pointing to a non-existent
+//     certificate provider instance
+//
+// The test verifies that RPCs to the first two clusters succeed, while RPCs to
+// the third cluster fails with an appropriate code and error message.
+func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) {
+	// Spin up an xDS management server. This uses a bootstrap config with a
+	// certificate provider instance name e2e.ClientSideCertProviderInstance.
+	mgmtServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+	defer cleanup()
+
+	// Create test backends for all three clusters
+	// backend1 configured with TLS creds, represents cluster1
+	// backend2 configured with insecure creds, represents cluster2
+	// backend3 configured with insecure creds, represents cluster3
+	creds := e2e.CreateServerTLSCredentials(t)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(creds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const clusterName3 = "cluster3-" + serviceName
+	const endpointsName1 = "endpoints1-" + serviceName
+	const endpointsName2 = "endpoints2-" + serviceName
+	const endpointsName3 = "endpoints3-" + serviceName
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
+					}},
+				},
+			},
+		}},
+	}}
+	// Clusters:
+	// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance.
+	// - cluster2 with no security configuration.
+	// - cluster3 with non-existent cert provider name.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
+		func() *v3clusterpb.Cluster {
+			cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
+			cluster3.TransportSocket = &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+								ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName: "non-existent-certificate-provider-instance-name",
+								},
+							},
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: "non-existent-certificate-provider-instance-name",
+							},
+						},
+					}),
+				},
+			}
+			return cluster3
+		}(),
+	}
+	// Endpoints for each of the above clusters with backends created earlier.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an RPC to be routed to cluster1 and verify that it succeeds.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server1.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+
+	}
+
+	// Make an RPC to be routed to cluster2 and verify that it succeeds.
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("UnaryCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server2.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// Make an RPC to be routed to cluster3 and verify that it fails.
+	const wantErr = `identitiy certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
+	if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}